From 5b75d3a904a2d9e813fde63caf79b9f9db6be4e7 Mon Sep 17 00:00:00 2001 From: Liam Date: Sun, 14 May 2023 22:13:37 +1000 Subject: [PATCH] enhance 3 regions deployment (#259) * enhance 3 regions deployment * remove other regions config * remove other unused code --- cicd/devnet/start.sh | 2 +- cicd/devnet/terraform/.env | 15 +- cicd/devnet/terraform/.terraform.lock.hcl | 39 ----- cicd/devnet/terraform/iam.tf | 28 ++++ cicd/devnet/terraform/main.tf | 136 ++++++------------ .../module/region/container-definition.tpl | 43 ++++++ .../terraform/{ => module/region}/ecs.tf | 19 +-- .../terraform/{ => module/region}/efs.tf | 6 +- cicd/devnet/terraform/module/region/main.tf | 97 +++++++++++++ .../terraform/module/region/variables.tf | 19 +++ cicd/devnet/terraform/s3.tf | 9 +- cicd/devnet/terraform/variables.tf | 37 ++++- 12 files changed, 289 insertions(+), 161 deletions(-) delete mode 100644 cicd/devnet/terraform/.terraform.lock.hcl create mode 100644 cicd/devnet/terraform/iam.tf create mode 100644 cicd/devnet/terraform/module/region/container-definition.tpl rename cicd/devnet/terraform/{ => module/region}/ecs.tf (84%) rename cicd/devnet/terraform/{ => module/region}/efs.tf (93%) create mode 100644 cicd/devnet/terraform/module/region/main.tf create mode 100644 cicd/devnet/terraform/module/region/variables.tf diff --git a/cicd/devnet/start.sh b/cicd/devnet/start.sh index 3565acf1e6..2719543dc9 100755 --- a/cicd/devnet/start.sh +++ b/cicd/devnet/start.sh @@ -55,7 +55,7 @@ echo "Starting nodes with $bootnodes ..." # Note: --gcmode=archive means node will store all historical data. This will lead to high memory usage. Only needed if you need the node to perform historical operations XDC --ethstats ${netstats} --gcmode=full \ --nat extip:${INSTANCE_IP} \ ---bootnodes ${bootnodes} --syncmode full \ +--bootnodes ${bootnodes} --syncmode fast \ --datadir /work/xdcchain --networkid 551 \ -port 30303 --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \ --rpcport 8545 \ diff --git a/cicd/devnet/terraform/.env b/cicd/devnet/terraform/.env index 09c4d68b40..a1681f393d 100644 --- a/cicd/devnet/terraform/.env +++ b/cicd/devnet/terraform/.env @@ -1,2 +1,13 @@ -num_of_nodes=0 -log_level=3 \ No newline at end of file +log_level=3 + +# Ohio +us-east-2_start=0 +us-east-2_end=36 + +# Ireland +eu-west-1_start=37 +eu-west-1_end=72 + +# Sydney +ap-southeast-2_start=73 +ap-southeast-2_end=110 diff --git a/cicd/devnet/terraform/.terraform.lock.hcl b/cicd/devnet/terraform/.terraform.lock.hcl deleted file mode 100644 index 97691fad56..0000000000 --- a/cicd/devnet/terraform/.terraform.lock.hcl +++ /dev/null @@ -1,39 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/aws" { - version = "4.32.0" - constraints = "~> 4.16" - hashes = [ - "h1:d4aUL6/J+BFhh1/Nh2rgctt+dqf07H9PipRn297hIIo=", - "zh:062c30cd8bcf29f8ee34c2b2509e4e8695c2bcac8b7a8145e1c72e83d4e68b13", - "zh:1503fabaace96a7eea4d73ced36a02a75ec587760850e58162e7eff419dcbb31", - "zh:39a1fa36f8cb999f048bf0000d9dab40b8b0c77df35584fb08aa8bd6c5052dee", - "zh:471a755d43b51cd7be3e386cebc151ad8d548c5dea798343620476887e721882", - "zh:61ed56fab811e62b8286e606d003f7eeb7e940ef99bb49c1d283d91c0b748cc7", - "zh:80607dfe5f7770d136d5c451308b9861084ffad08139de8014e48672ec43ea3f", - "zh:863bf0a6576f7a969a89631525250d947fbb207d3d13e7ca4f74d86bd97cdda3", - "zh:9a8f2e77e4f99dbb618eb8ad17218a4698833754b50d46da5727323a2050a400", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:9b74ff6e638c2a470b3599d57c2081e0095976da0a54b6590884d571f930b53b", - "zh:da4fc553d50ae833d860ec95120e271c29b4cb636917ab5991327362b7486bb7", - "zh:f4b86e7df4e846a38774e8e648b41c5ebaddcefa913cfa1864568086b7735575", - ] -} - -provider "registry.terraform.io/hashicorp/template" { - version = "2.2.0" - hashes = [ - "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", - "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", - "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", - "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", - "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", - "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", - "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", - "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", - "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", - "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", - "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", - ] -} diff --git a/cicd/devnet/terraform/iam.tf b/cicd/devnet/terraform/iam.tf new file mode 100644 index 0000000000..5a9e984a38 --- /dev/null +++ b/cicd/devnet/terraform/iam.tf @@ -0,0 +1,28 @@ +# IAM policies +data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +# Create the role +resource "aws_iam_role" "devnet_xdc_ecs_tasks_execution_role" { + name = "devnet-xdc-ecs-task-execution-role" + assume_role_policy = "${data.aws_iam_policy_document.xdc_ecs_tasks_execution_role.json}" +} + +# Attached the AWS managed policies to the new role +resource "aws_iam_role_policy_attachment" "devnet_xdc_ecs_tasks_execution_role" { + for_each = toset([ + "arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess", + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + "arn:aws:iam::aws:policy/AmazonElasticFileSystemsUtils" + ]) + role = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.name + policy_arn = each.value +} diff --git a/cicd/devnet/terraform/main.tf b/cicd/devnet/terraform/main.tf index 5b643347ec..b5fc04f531 100644 --- a/cicd/devnet/terraform/main.tf +++ b/cicd/devnet/terraform/main.tf @@ -9,114 +9,58 @@ terraform { required_version = ">= 1.2.0" } +# Default provider "aws" { region = "us-east-1" } -resource "aws_vpc" "devnet_vpc" { - cidr_block = "10.0.0.0/16" - instance_tenancy = "default" - enable_dns_hostnames = true - - tags = { - Name = "TfDevnetVpc" +provider "aws" { + alias = "us-east-2" + region = "us-east-2" +} + +module "us-east-2" { + source = "./module/region" + region = "us-east-2" + devnetNodeKeys = local.devnetNodeKeys["us-east-2"] + logLevel = local.logLevel + devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn + + providers = { + aws = aws.us-east-2 } } -resource "aws_subnet" "devnet_subnet" { - vpc_id = aws_vpc.devnet_vpc.id - cidr_block = "10.0.0.0/20" - map_public_ip_on_launch = true - availability_zone = "us-east-1a" - - tags = { - Name = "TfDevnetVpcSubnet" +provider "aws" { + alias = "eu-west-1" + region = "eu-west-1" +} + +module "eu-west-1" { + source = "./module/region" + region = "eu-west-1" + devnetNodeKeys = local.devnetNodeKeys["eu-west-1"] + logLevel = local.logLevel + devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn + + providers = { + aws = aws.eu-west-1 } } -resource "aws_internet_gateway" "devnet_gatewat" { - vpc_id = aws_vpc.devnet_vpc.id - - tags = { - Name = "TfDevnetGateway" - } +provider "aws" { + alias = "ap-southeast-2" + region = "ap-southeast-2" } -resource "aws_route_table" "devnet_route_table" { - vpc_id = aws_vpc.devnet_vpc.id +module "ap-southeast-2" { + source = "./module/region" + region = "ap-southeast-2" + devnetNodeKeys = local.devnetNodeKeys["ap-southeast-2"] + logLevel = local.logLevel + devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.devnet_gatewat.id - } - - tags = { - Name = "TfDevnetVpcRoutingTable" + providers = { + aws = aws.ap-southeast-2 } } - -resource "aws_route_table_association" "devnet_route_table_association" { - subnet_id = aws_subnet.devnet_subnet.id - route_table_id = aws_route_table.devnet_route_table.id -} - -resource "aws_default_security_group" "devnet_xdcnode_security_group" { - vpc_id = aws_vpc.devnet_vpc.id - - ingress { - from_port = 0 - to_port = 0 - protocol = -1 - cidr_blocks = ["0.0.0.0/0"] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - tags = { - Name = "TfDevnetNode" - } -} - -# IAM policies -data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ecs-tasks.amazonaws.com"] - } - } -} - -# Create the role -resource "aws_iam_role" "devnet_xdc_ecs_tasks_execution_role" { - name = "devnet-xdc-ecs-task-execution-role" - assume_role_policy = "${data.aws_iam_policy_document.xdc_ecs_tasks_execution_role.json}" -} - -# Attached the AWS managed policies to the new role -resource "aws_iam_role_policy_attachment" "devnet_xdc_ecs_tasks_execution_role" { - for_each = toset([ - "arn:aws:iam::aws:policy/AmazonElasticFileSystemClientFullAccess", - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", - "arn:aws:iam::aws:policy/AmazonElasticFileSystemsUtils" - ]) - role = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.name - policy_arn = each.value -} - -# Logs -resource "aws_cloudwatch_log_group" "devnet_cloud_watch_group" { - for_each = local.devnetNodeKyes - - name = "tf-${each.key}" - retention_in_days = 14 # Logs are only kept for 14 days - tags = { - Name = "TfDevnetCloudWatchGroup${each.key}" - } -} \ No newline at end of file diff --git a/cicd/devnet/terraform/module/region/container-definition.tpl b/cicd/devnet/terraform/module/region/container-definition.tpl new file mode 100644 index 0000000000..ae1fcc227f --- /dev/null +++ b/cicd/devnet/terraform/module/region/container-definition.tpl @@ -0,0 +1,43 @@ +[ + { + "name": "tfXdcNode", + "image": "xinfinorg/${xdc_environment}:${image_tag}", + "environment": [ + {"name": "PRIVATE_KEYS", "value": "${private_keys}"}, + {"name": "LOG_LEVEL", "value": "${log_level}"}, + {"name": "NODE_NAME", "value": "${node_name}"} + ], + "essential": true, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "${cloudwatch_group}", + "awslogs-region": "${cloudwatch_region}", + "awslogs-stream-prefix": "ecs" + } + }, + "portMappings": [ + { + "hostPort": 8555, + "protocol": "tcp", + "containerPort": 8555 + }, + { + "hostPort": 8545, + "protocol": "tcp", + "containerPort": 8545 + }, + { + "hostPort": 30303, + "protocol": "tcp", + "containerPort": 30303 + } + ], + "mountPoints": [ + { + "containerPath": "/work/xdcchain", + "sourceVolume": "efs" + } + ] + } +] \ No newline at end of file diff --git a/cicd/devnet/terraform/ecs.tf b/cicd/devnet/terraform/module/region/ecs.tf similarity index 84% rename from cicd/devnet/terraform/ecs.tf rename to cicd/devnet/terraform/module/region/ecs.tf index deeb56065a..de60938eeb 100644 --- a/cicd/devnet/terraform/ecs.tf +++ b/cicd/devnet/terraform/module/region/ecs.tf @@ -1,5 +1,5 @@ data template_file devnet_container_definition { - for_each = local.devnetNodeKyes + for_each = var.devnetNodeKeys template = "${file("${path.module}/container-definition.tpl")}" vars = { @@ -8,19 +8,20 @@ data template_file devnet_container_definition { node_name = "${each.key}" private_keys = "${each.value.pk}" cloudwatch_group = "tf-${each.key}" - log_level = "${lookup(each.value, "logLevel", "${local.logLevel}")}" + cloudwatch_region = "${var.region}" + log_level = "${lookup(each.value, "logLevel", "${var.logLevel}")}" } } resource "aws_ecs_task_definition" "devnet_task_definition_group" { - for_each = local.devnetNodeKyes + for_each = var.devnetNodeKeys family = "devnet-${each.key}" requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" container_definitions = data.template_file.devnet_container_definition[each.key].rendered - execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn - task_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn + execution_role_arn = var.devnet_xdc_ecs_tasks_execution_role_arn + task_role_arn = var.devnet_xdc_ecs_tasks_execution_role_arn # New nodes will consume a lot more CPU usage than existing nodes. # This is due to sync is resource heavy. Recommending set to below if doing sync: @@ -29,7 +30,7 @@ resource "aws_ecs_task_definition" "devnet_task_definition_group" { # cpu = 256 # memory = 2048 cpu = 512 - memory = 4096 + memory = 3072 volume { name = "efs" @@ -50,7 +51,7 @@ resource "aws_ecs_task_definition" "devnet_task_definition_group" { } data "aws_ecs_task_definition" "devnet_ecs_task_definition" { - for_each = local.devnetNodeKyes + for_each = var.devnetNodeKeys task_definition = aws_ecs_task_definition.devnet_task_definition_group[each.key].family } @@ -62,7 +63,7 @@ resource "aws_ecs_cluster" "devnet_ecs_cluster" { } resource "aws_ecs_service" "devnet_ecs_service" { - for_each = local.devnetNodeKyes + for_each = var.devnetNodeKeys name = "ecs-service-${each.key}" cluster = aws_ecs_cluster.devnet_ecs_cluster.id task_definition = "${aws_ecs_task_definition.devnet_task_definition_group[each.key].family}:${max(aws_ecs_task_definition.devnet_task_definition_group[each.key].revision, data.aws_ecs_task_definition.devnet_ecs_task_definition[each.key].revision)}" @@ -70,6 +71,8 @@ resource "aws_ecs_service" "devnet_ecs_service" { scheduling_strategy = "REPLICA" desired_count = 1 force_new_deployment = true + deployment_minimum_healthy_percent = 0 + deployment_maximum_percent = 100 network_configuration { subnets = [aws_subnet.devnet_subnet.id] diff --git a/cicd/devnet/terraform/efs.tf b/cicd/devnet/terraform/module/region/efs.tf similarity index 93% rename from cicd/devnet/terraform/efs.tf rename to cicd/devnet/terraform/module/region/efs.tf index f4dabe5e29..d03adc85b7 100644 --- a/cicd/devnet/terraform/efs.tf +++ b/cicd/devnet/terraform/module/region/efs.tf @@ -24,7 +24,7 @@ resource "aws_security_group" "devnet_efs_security_group" { } resource "aws_efs_file_system" "devnet_efs" { - for_each = local.devnetNodeKyes + for_each = var.devnetNodeKeys creation_token = "efs-${each.key}" performance_mode = "generalPurpose" throughput_mode = "bursting" @@ -38,14 +38,14 @@ resource "aws_efs_file_system" "devnet_efs" { } resource "aws_efs_mount_target" "devnet_efs_efs_mount_target" { - for_each = local.devnetNodeKyes + for_each = var.devnetNodeKeys file_system_id = aws_efs_file_system.devnet_efs[each.key].id subnet_id = aws_subnet.devnet_subnet.id security_groups = [aws_security_group.devnet_efs_security_group.id] } resource "aws_efs_access_point" "devnet_efs_access_point" { - for_each = local.devnetNodeKyes + for_each = var.devnetNodeKeys file_system_id = aws_efs_file_system.devnet_efs[each.key].id root_directory { path = "/${each.key}/database" diff --git a/cicd/devnet/terraform/module/region/main.tf b/cicd/devnet/terraform/module/region/main.tf new file mode 100644 index 0000000000..08c78c34ec --- /dev/null +++ b/cicd/devnet/terraform/module/region/main.tf @@ -0,0 +1,97 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.16" + } + } + + required_version = ">= 1.2.0" +} + +resource "aws_vpc" "devnet_vpc" { + cidr_block = "10.0.0.0/16" + instance_tenancy = "default" + enable_dns_hostnames = true + + tags = { + Name = "TfDevnetVpc" + } +} + +resource "aws_subnet" "devnet_subnet" { + vpc_id = aws_vpc.devnet_vpc.id + cidr_block = "10.0.0.0/20" + map_public_ip_on_launch = true + + tags = { + Name = "TfDevnetVpcSubnet" + } +} + +resource "aws_internet_gateway" "devnet_gatewat" { + vpc_id = aws_vpc.devnet_vpc.id + + tags = { + Name = "TfDevnetGateway" + } +} + +resource "aws_route_table" "devnet_route_table" { + vpc_id = aws_vpc.devnet_vpc.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.devnet_gatewat.id + } + + tags = { + Name = "TfDevnetVpcRoutingTable" + } +} + +resource "aws_route_table_association" "devnet_route_table_association" { + subnet_id = aws_subnet.devnet_subnet.id + route_table_id = aws_route_table.devnet_route_table.id +} + +resource "aws_default_security_group" "devnet_xdcnode_security_group" { + vpc_id = aws_vpc.devnet_vpc.id + + ingress { + description = "listener port" + from_port = 30303 + to_port = 30303 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "discovery port" + from_port = 30303 + to_port = 30303 + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + tags = { + Name = "TfDevnetNode" + } +} + +# Logs +resource "aws_cloudwatch_log_group" "devnet_cloud_watch_group" { + for_each = var.devnetNodeKeys + + name = "tf-${each.key}" + retention_in_days = 14 # Logs are only kept for 14 days + tags = { + Name = "TfDevnetCloudWatchGroup${each.key}" + } +} \ No newline at end of file diff --git a/cicd/devnet/terraform/module/region/variables.tf b/cicd/devnet/terraform/module/region/variables.tf new file mode 100644 index 0000000000..53d292f302 --- /dev/null +++ b/cicd/devnet/terraform/module/region/variables.tf @@ -0,0 +1,19 @@ +variable "region" { + description = "AWS region" + type = string +} + +variable "devnetNodeKeys" { + description = "each miner's key" + type = map +} + +variable "logLevel" { + description = "containers log level" + type = string +} + +variable "devnet_xdc_ecs_tasks_execution_role_arn" { + description = "aws iam role resource arn" + type = string +} \ No newline at end of file diff --git a/cicd/devnet/terraform/s3.tf b/cicd/devnet/terraform/s3.tf index 29820c995a..c7aba085ac 100644 --- a/cicd/devnet/terraform/s3.tf +++ b/cicd/devnet/terraform/s3.tf @@ -1,17 +1,14 @@ - - - # Bucket need to be created first. If first time run terraform init, need to comment out the below section terraform { backend "s3" { bucket = "tf-devnet-bucket" // This name need to be updated to be the same as local.s3BucketName. We can't use variable here. - key = "tf/terraform.tfstate" + key = "tf/terraform_new.tfstate" region = "us-east-1" encrypt = true } } -data "aws_s3_bucket_object" "devnet_xdc_node_config" { +data "aws_s3_object" "devnet_xdc_node_config" { bucket = local.s3BucketName key = "node-config.json" -} \ No newline at end of file +} diff --git a/cicd/devnet/terraform/variables.tf b/cicd/devnet/terraform/variables.tf index a3c7c59c5d..0af8d86077 100644 --- a/cicd/devnet/terraform/variables.tf +++ b/cicd/devnet/terraform/variables.tf @@ -7,13 +7,38 @@ locals { ... any other configuration we want to pass. } Note: No `n` is allowed in the node name - **/ - predefinedNodesConfig = jsondecode(data.aws_s3_bucket_object.devnet_xdc_node_config.body) + **/ + predefinedNodesConfig = jsondecode(data.aws_s3_object.devnet_xdc_node_config.body) envs = { for tuple in regexall("(.*)=(.*)", file(".env")) : tuple[0] => tuple[1] } logLevel = local.envs["log_level"] - keyNames =[for i in range(tonumber(local.envs["num_of_nodes"])) : "xdc${i}"] - devnetNodeKyes = { - for i in local.keyNames: i => local.predefinedNodesConfig[i] + + regions = [ + { + "name": "us-east-2", // Ohio + "start": local.envs["us-east-2_start"], + "end": local.envs["us-east-2_end"], + }, + { + "name": "eu-west-1", // Ireland + "start": local.envs["eu-west-1_start"], + "end": local.envs["eu-west-1_end"], + }, + { + "name": "ap-southeast-2", // Sydney + "start": local.envs["ap-southeast-2_start"], + "end": local.envs["ap-southeast-2_end"], + } + ] + + keyNames = { + for r in local.regions : + r.name => [for i in range(r.start, r.end+1) : "xdc${i}"] } + + devnetNodeKeys = { + for r in local.regions : + r.name => { for i in local.keyNames[r.name]: i => local.predefinedNodesConfig[i] } + } + s3BucketName = "tf-devnet-bucket" -} \ No newline at end of file +}