From d24f187bfdba212fd1d29ebe858d6968c6430d5a Mon Sep 17 00:00:00 2001 From: Banana-J Date: Mon, 15 Jan 2024 20:53:48 +1100 Subject: [PATCH] Host rpc node in ecs (#391) --- .gitignore | 3 +- .travis.yml | 1 + cicd/devnet/terraform/.env | 2 +- cicd/devnet/terraform/main.tf | 23 +++- cicd/devnet/terraform/module/region/ecs.tf | 8 +- cicd/devnet/terraform/module/region/main.tf | 8 ++ cicd/devnet/terraform/module/region/rpc.tf | 104 ++++++++++++++++++ .../terraform/module/region/variables.tf | 6 + cicd/devnet/terraform/variables.tf | 4 +- 9 files changed, 150 insertions(+), 9 deletions(-) create mode 100644 cicd/devnet/terraform/module/region/rpc.tf diff --git a/.gitignore b/.gitignore index b9ae71f5f0..5548c132dd 100644 --- a/.gitignore +++ b/.gitignore @@ -52,4 +52,5 @@ coverage.txt go.sum cicd/devnet/terraform/.terraform* cicd/devnet/tmp -.env \ No newline at end of file +.env +cicd/devnet/terraform/node-config.json \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 42f10018a4..b4e3db53b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -171,6 +171,7 @@ jobs: echo "Force deploy xdc-$i" aws ecs update-service --region ap-southeast-2 --cluster devnet-xdcnode-cluster --service ecs-service-xdc$i --force-new-deployment --no-cli-pager; done + aws ecs update-service --region ap-southeast-1 --cluster devnet-xdcnode-cluster --service ecs-service-rpc1 --force-new-deployment --no-cli-pager; - stage: (Devnet) Send Deployment Notification if: branch = dev-upgrade AND type = push AND tag IS blank diff --git a/cicd/devnet/terraform/.env b/cicd/devnet/terraform/.env index 0e3b90e588..4eb6ca5a95 100644 --- a/cicd/devnet/terraform/.env +++ b/cicd/devnet/terraform/.env @@ -10,4 +10,4 @@ eu_west_1_end=72 # Sydney ap_southeast_2_start=73 -ap_southeast_2_end=110 +ap_southeast_2_end=108 \ No newline at end of file diff --git a/cicd/devnet/terraform/main.tf b/cicd/devnet/terraform/main.tf index 473901163c..285b9fc010 100644 --- a/cicd/devnet/terraform/main.tf +++ b/cicd/devnet/terraform/main.tf @@ -23,7 +23,6 @@ module "us-east-2" { devnetNodeKeys = local.devnetNodeKeys["us-east-2"] logLevel = local.logLevel devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn - providers = { aws = aws.us-east-2 } @@ -40,7 +39,6 @@ module "eu-west-1" { devnetNodeKeys = local.devnetNodeKeys["eu-west-1"] logLevel = local.logLevel devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn - providers = { aws = aws.eu-west-1 } @@ -57,8 +55,27 @@ module "ap-southeast-2" { devnetNodeKeys = local.devnetNodeKeys["ap-southeast-2"] logLevel = local.logLevel devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn - providers = { aws = aws.ap-southeast-2 } } + +# WARNING: APSE-1 will only be used to host rpc node +# Workaround to avoid conflicts with existing ecs cluster in existing regions +provider "aws" { + alias = "ap-southeast-1" + region = "ap-southeast-1" +} + +module "ap-southeast-1-rpc" { + source = "./module/region" + region = "ap-southeast-1" + devnetNodeKeys = local.rpcNodeKeys + enableFixedIp = true + logLevel = local.logLevel + devnet_xdc_ecs_tasks_execution_role_arn = aws_iam_role.devnet_xdc_ecs_tasks_execution_role.arn + + providers = { + aws = aws.ap-southeast-1 + } +} diff --git a/cicd/devnet/terraform/module/region/ecs.tf b/cicd/devnet/terraform/module/region/ecs.tf index 6b691d2d16..0f589f9447 100644 --- a/cicd/devnet/terraform/module/region/ecs.tf +++ b/cicd/devnet/terraform/module/region/ecs.tf @@ -56,15 +56,17 @@ data "aws_ecs_task_definition" "devnet_ecs_task_definition" { task_definition = aws_ecs_task_definition.devnet_task_definition_group[each.key].family } +# ECS cluster resource "aws_ecs_cluster" "devnet_ecs_cluster" { - name = "devnet-xdcnode-cluster" - tags = { + name = "devnet-xdcnode-cluster" + tags = { Name = "TfDevnetEcsCluster" } } + resource "aws_ecs_service" "devnet_ecs_service" { - for_each = var.devnetNodeKeys + for_each = var.enableFixedIp ? {} : var.devnetNodeKeys name = "ecs-service-${each.key}" cluster = aws_ecs_cluster.devnet_ecs_cluster.id task_definition = "${aws_ecs_task_definition.devnet_task_definition_group[each.key].family}:${max(aws_ecs_task_definition.devnet_task_definition_group[each.key].revision, data.aws_ecs_task_definition.devnet_ecs_task_definition[each.key].revision)}" diff --git a/cicd/devnet/terraform/module/region/main.tf b/cicd/devnet/terraform/module/region/main.tf index 3079b17f23..8e2ac0dd55 100644 --- a/cicd/devnet/terraform/module/region/main.tf +++ b/cicd/devnet/terraform/module/region/main.tf @@ -72,6 +72,14 @@ resource "aws_default_security_group" "devnet_xdcnode_security_group" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + description = "rpc port" + from_port = 8545 + to_port = 8545 + protocol = "tcp" + cidr_blocks = ["10.0.0.0/16"] + } + egress { from_port = 0 to_port = 0 diff --git a/cicd/devnet/terraform/module/region/rpc.tf b/cicd/devnet/terraform/module/region/rpc.tf new file mode 100644 index 0000000000..2ba07813c1 --- /dev/null +++ b/cicd/devnet/terraform/module/region/rpc.tf @@ -0,0 +1,104 @@ +# Allocate an Elastic IP for the NLB +resource "aws_eip" "nlb_eip" { + domain = "vpc" +} + + +# Create a Network Load Balancer +resource "aws_lb" "rpc_node_nlb" { + count = var.enableFixedIp ? 1 : 0 + name = "rpc-node-nlb" + load_balancer_type = "network" + + enable_deletion_protection = false + + subnet_mapping { + subnet_id = aws_subnet.devnet_subnet.id + allocation_id = aws_eip.nlb_eip.id + } +} + +# Listener and Target Group for the rpc node container +resource "aws_lb_target_group" "rpc_node_tg_8545" { + count = var.enableFixedIp ? 1 : 0 + name = "rpc-node-tg" + port = 8545 + protocol = "TCP" + vpc_id = aws_vpc.devnet_vpc.id + target_type = "ip" +} + +resource "aws_lb_listener" "rpc_node_listener_8545" { + count = var.enableFixedIp ? 1 : 0 + load_balancer_arn = aws_lb.rpc_node_nlb[0].arn + port = 8545 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.rpc_node_tg_8545[0].arn + } +} + +resource "aws_ecs_service" "devnet_rpc_node_ecs_service" { + for_each = var.enableFixedIp ? var.devnetNodeKeys : {} + name = "ecs-service-${each.key}" + cluster = aws_ecs_cluster.devnet_ecs_cluster.id + task_definition = "${aws_ecs_task_definition.devnet_task_definition_group[each.key].family}:${max(aws_ecs_task_definition.devnet_task_definition_group[each.key].revision, data.aws_ecs_task_definition.devnet_ecs_task_definition[each.key].revision)}" + launch_type = "FARGATE" + scheduling_strategy = "REPLICA" + desired_count = 1 + force_new_deployment = true + deployment_minimum_healthy_percent = 0 + deployment_maximum_percent = 100 + + network_configuration { + subnets = [aws_subnet.devnet_subnet.id] + assign_public_ip = true + security_groups = [ + aws_default_security_group.devnet_xdcnode_security_group.id + ] + } + + deployment_circuit_breaker { + enable = true + rollback = false + } + + load_balancer { + target_group_arn = aws_lb_target_group.rpc_node_tg_8545[0].arn + container_name = "tfXdcNode" + container_port = 8545 + } + + depends_on = [ + aws_lb_listener.rpc_node_listener_8545 + ] + + tags = { + Name = "TfDevnetRpcNodeEcsService-${each.key}" + } +} + +# Target Group for port 30303 +resource "aws_lb_target_group" "rpc_node_tg_30303" { + count = var.enableFixedIp ? 1 : 0 + name = "rpc-node-tg-30303" + port = 30303 + protocol = "TCP" + vpc_id = aws_vpc.devnet_vpc.id + target_type = "ip" +} + +# Listener for port 30303 +resource "aws_lb_listener" "rpc_node_listener_30303" { + count = var.enableFixedIp ? 1 : 0 + load_balancer_arn = aws_lb.rpc_node_nlb[0].arn + port = 30303 + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.rpc_node_tg_30303[0].arn + } +} \ No newline at end of file diff --git a/cicd/devnet/terraform/module/region/variables.tf b/cicd/devnet/terraform/module/region/variables.tf index 53d292f302..fc7afca84b 100644 --- a/cicd/devnet/terraform/module/region/variables.tf +++ b/cicd/devnet/terraform/module/region/variables.tf @@ -16,4 +16,10 @@ variable "logLevel" { variable "devnet_xdc_ecs_tasks_execution_role_arn" { description = "aws iam role resource arn" type = string +} + +variable "enableFixedIp" { + description = "a flag to indicate whether fixed ip should be associated to the nodes. This is used for RPC node" + type = bool + default = false } \ No newline at end of file diff --git a/cicd/devnet/terraform/variables.tf b/cicd/devnet/terraform/variables.tf index c86ff7df48..b4c4b14faf 100644 --- a/cicd/devnet/terraform/variables.tf +++ b/cicd/devnet/terraform/variables.tf @@ -5,7 +5,7 @@ locals { {{Name of the node, in a pattern of 'xdc'+ number. i.e xdc50}}: { pk: {{Value of the node private key}}, ... any other configuration we want to pass. - } + } Note: No `n` is allowed in the node name **/ predefinedNodesConfig = jsondecode(data.aws_s3_object.devnet_xdc_node_config.body) @@ -39,6 +39,8 @@ locals { for r in local.regions : r.name => { for i in local.keyNames[r.name]: i => local.predefinedNodesConfig[i] } } + + rpcNodeKeys = { "rpc1": local.predefinedNodesConfig["rpc1"]} // we hardcode the rpc to a single node for now s3BucketName = "tf-devnet-bucket" }