enhance 3 regions deployment (#259)

* enhance 3 regions deployment

* remove other regions config

* remove other unused code

cicd/devnet/start.sh

@@ -55,7 +55,7 @@ echo "Starting nodes with $bootnodes ..."
 # Note: --gcmode=archive means node will store all historical data. This will lead to high memory usage. Only needed if you need the node to perform historical operations
 XDC --ethstats ${netstats} --gcmode=full \
 --nat extip:${INSTANCE_IP} \
---bootnodes ${bootnodes} --syncmode full \
+--bootnodes ${bootnodes} --syncmode fast \
 --datadir /work/xdcchain --networkid 551 \
 -port 30303 --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \
 --rpcport 8545 \

cicd/devnet/terraform/.env

@@ -1,2 +1,13 @@
-num_of_nodes=0
-log_level=3
+log_level=3
+
+# Ohio
+us-east-2_start=0
+us-east-2_end=36
+
+# Ireland
+eu-west-1_start=37
+eu-west-1_end=72
+
+# Sydney
+ap-southeast-2_start=73
+ap-southeast-2_end=110

cicd/devnet/terraform/.terraform.lock.hcl

@@ -1,39 +0,0 @@
-# This file is maintained automatically by "terraform init".
-# Manual edits may be lost in future updates.
-
-provider "registry.terraform.io/hashicorp/aws" {
-  version     = "4.32.0"
-  constraints = "~> 4.16"
-  hashes = [
-    "h1:d4aUL6/J+BFhh1/Nh2rgctt+dqf07H9PipRn297hIIo=",
-    "zh:062c30cd8bcf29f8ee34c2b2509e4e8695c2bcac8b7a8145e1c72e83d4e68b13",
-    "zh:1503fabaace96a7eea4d73ced36a02a75ec587760850e58162e7eff419dcbb31",
-    "zh:39a1fa36f8cb999f048bf0000d9dab40b8b0c77df35584fb08aa8bd6c5052dee",
-    "zh:471a755d43b51cd7be3e386cebc151ad8d548c5dea798343620476887e721882",
-    "zh:61ed56fab811e62b8286e606d003f7eeb7e940ef99bb49c1d283d91c0b748cc7",
-    "zh:80607dfe5f7770d136d5c451308b9861084ffad08139de8014e48672ec43ea3f",
-    "zh:863bf0a6576f7a969a89631525250d947fbb207d3d13e7ca4f74d86bd97cdda3",
-    "zh:9a8f2e77e4f99dbb618eb8ad17218a4698833754b50d46da5727323a2050a400",
-    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
-    "zh:9b74ff6e638c2a470b3599d57c2081e0095976da0a54b6590884d571f930b53b",
-    "zh:da4fc553d50ae833d860ec95120e271c29b4cb636917ab5991327362b7486bb7",
-    "zh:f4b86e7df4e846a38774e8e648b41c5ebaddcefa913cfa1864568086b7735575",
-  ]
-}
-
-provider "registry.terraform.io/hashicorp/template" {
-  version = "2.2.0"
-  hashes = [
-    "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=",
-    "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386",
-    "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53",
-    "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603",
-    "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16",
-    "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776",
-    "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451",
-    "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae",
-    "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde",
-    "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d",
-    "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2",
-  ]
-}

cicd/devnet/terraform/iam.tf

@@ -0,0 +1,28 @@
+# IAM policies
+data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" {
+  statement {
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["ecs-tasks.amazonaws.com"]
+    }
+  }
+}
+
+# Create the role
+resource "aws_iam_role" "devnet_xdc_ecs_tasks_execution_role" {
+  name               = "devnet-xdc-ecs-task-execution-role"
+  assume_role_policy = "${data.aws_iam_policy_document.xdc_ecs_tasks_execution_role.json}"
+}
+
+# Attached the AWS managed policies to the new role
+resource "aws_iam_role_policy_attachment" "devnet_xdc_ecs_tasks_execution_role" {
+  for_each = toset([
+    "arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess", 
+    "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
+    "arn:aws:iam::aws:policy/AmazonElasticFileSystemsUtils"
+  ])
+  role       = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.name
+  policy_arn = each.value
+}

cicd/devnet/terraform/main.tf

@@ -9,114 +9,58 @@ terraform {
   required_version = ">= 1.2.0"
 }
 
+# Default
 provider "aws" {
   region  = "us-east-1"
 }
 
-resource "aws_vpc" "devnet_vpc" {
-  cidr_block = "10.0.0.0/16"
-  instance_tenancy = "default"
-  enable_dns_hostnames = true
-  
-  tags = {
-    Name = "TfDevnetVpc"
+provider "aws" {
+  alias = "us-east-2"
+  region  = "us-east-2"
+}
+
+module "us-east-2" {
+  source = "./module/region"
+  region = "us-east-2"
+  devnetNodeKeys = local.devnetNodeKeys["us-east-2"]
+  logLevel = local.logLevel
+  devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn
+
+  providers = {
+    aws = aws.us-east-2
   }
 }
 
-resource "aws_subnet" "devnet_subnet" {
-  vpc_id = aws_vpc.devnet_vpc.id
-  cidr_block = "10.0.0.0/20"
-  map_public_ip_on_launch = true
-  availability_zone = "us-east-1a"
-  
-  tags = {
-    Name = "TfDevnetVpcSubnet"
+provider "aws" {
+  alias = "eu-west-1"
+  region  = "eu-west-1"
+}
+
+module "eu-west-1" {
+  source = "./module/region"
+  region = "eu-west-1"
+  devnetNodeKeys = local.devnetNodeKeys["eu-west-1"]
+  logLevel = local.logLevel
+  devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn
+
+  providers = {
+    aws = aws.eu-west-1
   }
 }
 
-resource "aws_internet_gateway" "devnet_gatewat" {
-  vpc_id = aws_vpc.devnet_vpc.id
-
-  tags = {
-    Name = "TfDevnetGateway"
-  }
+provider "aws" {
+  alias = "ap-southeast-2"
+  region  = "ap-southeast-2"
 }
 
-resource "aws_route_table" "devnet_route_table" {
-  vpc_id = aws_vpc.devnet_vpc.id
+module "ap-southeast-2" {
+  source = "./module/region"
+  region = "ap-southeast-2"
+  devnetNodeKeys = local.devnetNodeKeys["ap-southeast-2"]
+  logLevel = local.logLevel
+  devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn
 
-  route {
-    cidr_block = "0.0.0.0/0"
-    gateway_id = aws_internet_gateway.devnet_gatewat.id
-  }
-
-  tags = {
-    Name = "TfDevnetVpcRoutingTable"
+  providers = {
+    aws = aws.ap-southeast-2
   }
 }
-
-resource "aws_route_table_association" "devnet_route_table_association" {
-  subnet_id      = aws_subnet.devnet_subnet.id
-  route_table_id = aws_route_table.devnet_route_table.id
-}
-
-resource "aws_default_security_group" "devnet_xdcnode_security_group" {
-  vpc_id = aws_vpc.devnet_vpc.id
-
-  ingress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = -1
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-
-  egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-  tags = {
-    Name = "TfDevnetNode"
-  }
-}
-
-# IAM policies
-data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" {
-  statement {
-    actions = ["sts:AssumeRole"]
-
-    principals {
-      type        = "Service"
-      identifiers = ["ecs-tasks.amazonaws.com"]
-    }
-  }
-}
-
-# Create the role
-resource "aws_iam_role" "devnet_xdc_ecs_tasks_execution_role" {
-  name               = "devnet-xdc-ecs-task-execution-role"
-  assume_role_policy = "${data.aws_iam_policy_document.xdc_ecs_tasks_execution_role.json}"
-}
-
-# Attached the AWS managed policies to the new role
-resource "aws_iam_role_policy_attachment" "devnet_xdc_ecs_tasks_execution_role" {
-  for_each = toset([
-    "arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess", 
-    "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy",
-    "arn:aws:iam::aws:policy/AmazonElasticFileSystemsUtils"
-  ])
-  role       = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.name
-  policy_arn = each.value
-}
-
-# Logs
-resource "aws_cloudwatch_log_group" "devnet_cloud_watch_group" {
-  for_each = local.devnetNodeKyes
-
-  name = "tf-${each.key}"
-  retention_in_days = 14 # Logs are only kept for 14 days
-  tags = {
-    Name = "TfDevnetCloudWatchGroup${each.key}"
-  }
-}

cicd/devnet/terraform/module/region/container-definition.tpl

@@ -0,0 +1,43 @@
+[
+  {
+    "name": "tfXdcNode",
+    "image": "xinfinorg/${xdc_environment}:${image_tag}",
+    "environment": [
+      {"name": "PRIVATE_KEYS", "value": "${private_keys}"},
+      {"name": "LOG_LEVEL", "value": "${log_level}"},
+      {"name": "NODE_NAME", "value": "${node_name}"}
+    ],
+    "essential": true,
+    "logConfiguration": {
+      "logDriver": "awslogs",
+      "options": {
+        "awslogs-group": "${cloudwatch_group}",
+        "awslogs-region": "${cloudwatch_region}",
+        "awslogs-stream-prefix": "ecs"
+      }
+    },
+    "portMappings": [
+      {
+        "hostPort": 8555,
+        "protocol": "tcp",
+        "containerPort": 8555
+      },
+      {
+        "hostPort": 8545,
+        "protocol": "tcp",
+        "containerPort": 8545
+      },
+      {
+        "hostPort": 30303,
+        "protocol": "tcp",
+        "containerPort": 30303
+      }
+    ],
+    "mountPoints": [
+      {
+        "containerPath": "/work/xdcchain",
+        "sourceVolume": "efs"
+      }
+    ]
+  }
+]

cicd/devnet/terraform/module/region/ecs.tf

@@ -1,5 +1,5 @@
 data template_file devnet_container_definition {
-  for_each = local.devnetNodeKyes
+  for_each = var.devnetNodeKeys
   template = "${file("${path.module}/container-definition.tpl")}"
 
   vars = {
@@ -8,19 +8,20 @@ data template_file devnet_container_definition {
     node_name = "${each.key}"
     private_keys = "${each.value.pk}"
     cloudwatch_group = "tf-${each.key}"
-    log_level = "${lookup(each.value, "logLevel", "${local.logLevel}")}"
+    cloudwatch_region = "${var.region}"
+    log_level = "${lookup(each.value, "logLevel", "${var.logLevel}")}"
   }
 }
 
 resource "aws_ecs_task_definition" "devnet_task_definition_group" {
-  for_each = local.devnetNodeKyes
+  for_each = var.devnetNodeKeys
   
   family = "devnet-${each.key}"
   requires_compatibilities = ["FARGATE"]
   network_mode = "awsvpc"
   container_definitions = data.template_file.devnet_container_definition[each.key].rendered
-  execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn
-  task_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn
+  execution_role_arn = var.devnet_xdc_ecs_tasks_execution_role_arn
+  task_role_arn = var.devnet_xdc_ecs_tasks_execution_role_arn
   
   # New nodes will consume a lot more CPU usage than existing nodes. 
   # This is due to sync is resource heavy. Recommending set to below if doing sync:
@@ -29,7 +30,7 @@ resource "aws_ecs_task_definition" "devnet_task_definition_group" {
   # cpu = 256
   # memory = 2048
   cpu = 512
-  memory = 4096
+  memory = 3072
   volume {
     name = "efs"
 
@@ -50,7 +51,7 @@ resource "aws_ecs_task_definition" "devnet_task_definition_group" {
 }
 
 data "aws_ecs_task_definition" "devnet_ecs_task_definition" {
-  for_each = local.devnetNodeKyes
+  for_each = var.devnetNodeKeys
   task_definition = aws_ecs_task_definition.devnet_task_definition_group[each.key].family
 }
 
@@ -62,7 +63,7 @@ resource "aws_ecs_cluster" "devnet_ecs_cluster" {
 }
 
 resource "aws_ecs_service" "devnet_ecs_service" {
-  for_each = local.devnetNodeKyes
+  for_each = var.devnetNodeKeys
   name                 = "ecs-service-${each.key}"
   cluster              = aws_ecs_cluster.devnet_ecs_cluster.id
   task_definition      = "${aws_ecs_task_definition.devnet_task_definition_group[each.key].family}:${max(aws_ecs_task_definition.devnet_task_definition_group[each.key].revision, data.aws_ecs_task_definition.devnet_ecs_task_definition[each.key].revision)}"
@@ -70,6 +71,8 @@ resource "aws_ecs_service" "devnet_ecs_service" {
   scheduling_strategy  = "REPLICA"
   desired_count        = 1
   force_new_deployment = true
+  deployment_minimum_healthy_percent = 0
+  deployment_maximum_percent = 100
 
   network_configuration {
     subnets          = [aws_subnet.devnet_subnet.id]

cicd/devnet/terraform/module/region/efs.tf

@@ -24,7 +24,7 @@ resource "aws_security_group" "devnet_efs_security_group" {
 }
 
 resource "aws_efs_file_system" "devnet_efs" {
-  for_each = local.devnetNodeKyes
+  for_each = var.devnetNodeKeys
   creation_token = "efs-${each.key}"
   performance_mode = "generalPurpose"
   throughput_mode = "bursting"
@@ -38,14 +38,14 @@ resource "aws_efs_file_system" "devnet_efs" {
  }
 
 resource "aws_efs_mount_target" "devnet_efs_efs_mount_target" {
-  for_each = local.devnetNodeKyes
+  for_each = var.devnetNodeKeys
   file_system_id = aws_efs_file_system.devnet_efs[each.key].id
   subnet_id      = aws_subnet.devnet_subnet.id
   security_groups = [aws_security_group.devnet_efs_security_group.id]
 }
 
 resource "aws_efs_access_point" "devnet_efs_access_point" {
-  for_each = local.devnetNodeKyes
+  for_each = var.devnetNodeKeys
   file_system_id = aws_efs_file_system.devnet_efs[each.key].id
   root_directory {
     path = "/${each.key}/database"

cicd/devnet/terraform/module/region/main.tf

@@ -0,0 +1,97 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.16"
+    }
+  }
+
+  required_version = ">= 1.2.0"
+}
+
+resource "aws_vpc" "devnet_vpc" {
+  cidr_block = "10.0.0.0/16"
+  instance_tenancy = "default"
+  enable_dns_hostnames = true
+  
+  tags = {
+    Name = "TfDevnetVpc"
+  }
+}
+
+resource "aws_subnet" "devnet_subnet" {
+  vpc_id = aws_vpc.devnet_vpc.id
+  cidr_block = "10.0.0.0/20"
+  map_public_ip_on_launch = true
+
+  tags = {
+    Name = "TfDevnetVpcSubnet"
+  }
+}
+
+resource "aws_internet_gateway" "devnet_gatewat" {
+  vpc_id = aws_vpc.devnet_vpc.id
+
+  tags = {
+    Name = "TfDevnetGateway"
+  }
+}
+
+resource "aws_route_table" "devnet_route_table" {
+  vpc_id = aws_vpc.devnet_vpc.id
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = aws_internet_gateway.devnet_gatewat.id
+  }
+
+  tags = {
+    Name = "TfDevnetVpcRoutingTable"
+  }
+}
+
+resource "aws_route_table_association" "devnet_route_table_association" {
+  subnet_id      = aws_subnet.devnet_subnet.id
+  route_table_id = aws_route_table.devnet_route_table.id
+}
+
+resource "aws_default_security_group" "devnet_xdcnode_security_group" {
+  vpc_id = aws_vpc.devnet_vpc.id
+
+  ingress {
+    description = "listener port"
+    from_port   = 30303
+    to_port     = 30303
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    description = "discovery port"
+    from_port   = 30303
+    to_port     = 30303
+    protocol    = "udp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  tags = {
+    Name = "TfDevnetNode"
+  }
+}
+
+# Logs
+resource "aws_cloudwatch_log_group" "devnet_cloud_watch_group" {
+  for_each = var.devnetNodeKeys
+
+  name = "tf-${each.key}"
+  retention_in_days = 14 # Logs are only kept for 14 days
+  tags = {
+    Name = "TfDevnetCloudWatchGroup${each.key}"
+  }
+}

cicd/devnet/terraform/module/region/variables.tf

@@ -0,0 +1,19 @@
+variable "region" {
+  description = "AWS region"
+  type        = string
+}
+
+variable "devnetNodeKeys" {
+  description = "each miner's key"
+  type        = map
+}
+
+variable "logLevel" {
+  description = "containers log level"
+  type        = string
+}
+
+variable "devnet_xdc_ecs_tasks_execution_role_arn" {
+  description = "aws iam role resource arn"
+  type        = string
+}

cicd/devnet/terraform/s3.tf

@@ -1,17 +1,14 @@
-
-
-
 # Bucket need to be created first. If first time run terraform init, need to comment out the below section
 terraform {
   backend "s3" {
     bucket = "tf-devnet-bucket" // This name need to be updated to be the same as local.s3BucketName. We can't use variable here.
-    key    = "tf/terraform.tfstate"
+    key    = "tf/terraform_new.tfstate"
     region = "us-east-1"
     encrypt = true
   }
 }
 
-data "aws_s3_bucket_object" "devnet_xdc_node_config" {
+data "aws_s3_object" "devnet_xdc_node_config" {
   bucket = local.s3BucketName
   key    = "node-config.json"
-}
+}

cicd/devnet/terraform/variables.tf

@@ -7,13 +7,38 @@ locals {
       ... any other configuration we want to pass.
     }
     Note: No `n` is allowed in the node name
-  **/
-    predefinedNodesConfig = jsondecode(data.aws_s3_bucket_object.devnet_xdc_node_config.body)
+    **/
+    predefinedNodesConfig = jsondecode(data.aws_s3_object.devnet_xdc_node_config.body)
     envs = { for tuple in regexall("(.*)=(.*)", file(".env")) : tuple[0] => tuple[1] }
     logLevel = local.envs["log_level"]
-    keyNames =[for i in range(tonumber(local.envs["num_of_nodes"])) : "xdc${i}"]
-    devnetNodeKyes = {
-      for i in local.keyNames: i => local.predefinedNodesConfig[i]
+
+    regions = [
+      {
+        "name": "us-east-2", // Ohio
+        "start": local.envs["us-east-2_start"],
+        "end": local.envs["us-east-2_end"],
+      },
+      {
+        "name": "eu-west-1", // Ireland
+        "start": local.envs["eu-west-1_start"],
+        "end": local.envs["eu-west-1_end"],
+      },
+      {
+        "name": "ap-southeast-2", // Sydney
+        "start": local.envs["ap-southeast-2_start"],
+        "end": local.envs["ap-southeast-2_end"],
+      }
+   ]
+
+    keyNames = {
+      for r in local.regions : 
+        r.name => [for i in range(r.start, r.end+1) : "xdc${i}"]
     }
+
+    devnetNodeKeys = {
+      for r in local.regions : 
+        r.name => { for i in local.keyNames[r.name]: i => local.predefinedNodesConfig[i] }
+    }
+
     s3BucketName = "tf-devnet-bucket"
-}
+}