Merge pull request #195 from XinFinOrg/XIN-243-finalise-terraform-for-devnet

finalise terraform for xdc devnet

.travis.yml

@@ -115,7 +115,22 @@ jobs:
         - terraform validate $tf_validation_cli_options
         - terraform plan $tf_plan_cli_options
   
-  - stage: (Devnet)Terraform apply
+  - stage: (Devnet) Build, and push images
+    if: branch = dev-upgrade AND type = push AND tag IS blank
+    services:
+      - docker
+    install: skip
+    before_script:
+      - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+      - docker --version  # document the version travis is using
+      - docker build -t xdc-devnet -f cicd/devnet/Dockerfile .
+    script:
+      - pip install --user awscli # install aws cli w/o sudo
+      - export PATH=$PATH:$HOME/.local/bin # put aws in the path
+      - docker tag xdc-devnet:latest xinfinorg/devnet:latest # Always push to the latest
+      - docker push xinfinorg/devnet:latest
+
+  - stage: Terraform apply
     if: branch = dev-upgrade AND type = push AND tag IS blank
     dist: xenial
     language: bash
@@ -131,22 +146,11 @@ jobs:
       # Terraform init and then apply changes to environment
       - terraform init $tf_init_cli_options
       - terraform apply $tf_apply_cli_options
-  
-  - stage: (Devnet) Build, push and deploy
-    if: branch = dev-upgrade AND type = push AND tag IS blank
-    services:
-      - docker
-    install: skip
-    before_script:
-      - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-      - docker --version  # document the version travis is using
-      - docker build -t xdc-devnet -f cicd/devnet/Dockerfile .
-    script:
-      - pip install --user awscli # install aws cli w/o sudo
-      - export PATH=$PATH:$HOME/.local/bin # put aws in the path
-      - docker tag xdc-devnet:latest xinfinorg/devnet:latest # Always push to the latest
-      - docker push xinfinorg/devnet:latest
-      - sleep 10 # Wait for 10 second before asking the ecs to update the images
-      - aws ecs update-service --region us-east-1 --cluster devnet --service devnet-group-1 --force-new-deployment #TODO: Temporary solution until we have proper automated scripts ready
-  
-  
+      - sleep 20
+      - |
+        source cicd/devnet/terraform/.env
+        for ((i=0;i<$num_of_nodes;i++)); do
+          echo "Force deploy xdc-$i"
+          sleep 5 && aws ecs update-service --region us-east-1 --cluster devnet-xdcnode-cluster --service ecs-service-xdc$i --force-new-deployment;
+        done
+

README.md

@@ -43,6 +43,10 @@ If at a later stage if some predecided amount of owners ( investors ) vote that
 
 ### For developers
 
+#### Continues integration & delivery
+See https://github.com/XinFinOrg/XDPoSChain/tree/dev-upgrade/cicd
+
+
 ### To contribute
 
 Simple create a pull request along with proper reasoning, we'll get back to you.

cicd/README.md

@@ -1,7 +1,10 @@
 # CI/CD pipeline for XDC
 This directory contains CI/CD scripts used for each of the XDC environments.
 
-### Devnet
+## How to deploy more nodes
+Adjust the number of variable `num_of_nodes` under file `.env`. (**Maximum supported is 58**)
+
+## Devnet
 Each PR merged into `dev-upgrade` will trigger below actions:
 - Tests
 - Terraform to apply infrascture changes(if any)
@@ -9,20 +12,33 @@ Each PR merged into `dev-upgrade` will trigger below actions:
 - Docker push to docker hub. https://hub.docker.com/repository/docker/xinfinorg/devnet
 - Deployment of the latest XDC image(from above) to devnet run by AWS ECS
 
-In order to allow pipeline able to push and deploy via ECR and ECS, we require below environment variables to be injected into the CI pipeline:
-1. ECR_REPO_NAME
-2. ECR_BASE_URI
-3. AWS_ACCESS_KEY_ID
-4. AWS_SECRET_ACCESS_KEY
+### First time set up an new environment
+1. Pre-generate a list of node private keys in below format
+```
+{
+  "xdc0": {
+    "pk": {{PRIVATE KEY}}
+  },
+  "xdc1": {...},
+  "xdc{{NUMBER}}: {...}
+}
+```
+2. Access to aws console, create a bucket with name `terraform-devnet-bucket`
+3. Upload the file from step 1 into the above bucket with name `node-config.json`
+4. In order to allow pipeline able to push and deploy via ECR and ECS, we require below environment variables to be injected into the CI pipeline:
+  1. DOCKER_USERNAME
+  2. DOCKER_PASSWORD
+  3. AWS_ACCESS_KEY_ID
+  4. AWS_SECRET_ACCESS_KEY
+  
+You are all set!
 
-#### How to spin up more nodes in devnet
-NOTE: The terraform managed auto deployment is still under deployment. The current best way to spin up new nodes is done by below:
-1. `docker pull xinfinorg/devnet:latest`
-2. `docker run -it -e PRIVATE_KEYS={{Wallet-Private-key-Here}} xinfinorg/devnet:latest`
+## Testnet
+*** WIP ***
+Testnet release build are triggered by cutting a "pre-release" tag which matches the name of `TESTNET-{{release-version}}` from dev-upgrade or master branch.
+An example can be found here: https://github.com/XinFinOrg/XDPoSChain/releases/tag/Testnet-v2.0.0
+For more information, refer to github documentation on the release: https://docs.github.com/en/repositories/releasing-projects-on-github/about-releases
 
-
-### Testnet
-**WIP**
-
-### Mainnet
-**WIP**
+## Mainnet
+*** WIP ***
+Mainnet release are triggered by making a normal release tag with name starting with `v` (stands for version) from the master branch.

cicd/devnet/Dockerfile

@@ -31,6 +31,6 @@ EXPOSE 8545
 # ws
 EXPOSE 8555
 # port
-EXPOSE 30304
+EXPOSE 30303
 
 ENTRYPOINT ["bash","/work/start.sh"]

cicd/devnet/bootnodes.list

@@ -1,2 +1,2 @@
-enode://1c20e6b46ce608c1fe739e78611225b94e663535b74a1545b1667eac8ff75ed43216306d123306c10e043f228e42cc53cb2728655019292380313393eaaf6e23@194.233.77.19:30301
-enode://1c20e6b46ce608c1fe739e78611225b94e663535b74a1545b1667eac8ff75ed43216306d123306c10e043f228e42cc53cb2728655019292380313393eaaf6e23@66.94.98.186:30301
+enode://c31bf87732529ef2d92e247f546fb139b8ab7476fc4e2d14fe4ce38631890c11ca9f87fed8c1b3e505e116d9a2674644bbef7ec296e518b688cc692e2aef885d@194.233.77.19:30303
+enode://a2e685e0daf718d4697f49eaa7c8e8bdfa25678d5b24afd8caa684261a90c513062b1fe9140effa04b7a98b0c971607b6413fe993344875e97137c00a7993efb@66.94.121.151:30303

cicd/devnet/start.sh

@@ -2,7 +2,7 @@
 
 echo "Preparing to start the XDC chain, it's likely to take up to 1 minute"
 # Sleep for > 30 as we need to wait for the ECS tasks container being killed by fargate. Otherwise it will ended up with two same nodes running on a single /work/xdcchain directory
-sleep 45
+sleep 60
 
 if [ ! -d /work/xdcchain/XDC/chaindata ]
 then
@@ -41,10 +41,11 @@ if test -z "$LOG_LEVEL"
 then
   echo "Log level not set, default to verbosity of 3"
 else
+  echo "Log level found, set to $LOG_LEVEL"
   log_level=$LOG_LEVEL
 fi
 
-netstats="aws_${wallet}:xinfin_xdpos_hybrid_network_stats@devnetstats.apothem.network:2000"
+netstats="${NODE_NAME}-${wallet}:xinfin_xdpos_hybrid_network_stats@devnetstats.apothem.network:2000"
 INSTANCE_IP=$(curl https://checkip.amazonaws.com)
 
 echo "Running a node with wallet: ${wallet} at IP: ${INSTANCE_IP}"
@@ -53,7 +54,7 @@ echo "Starting nodes with $bootnodes ..."
 XDC --ethstats ${netstats} --gcmode=archive \
 --bootnodes ${bootnodes} --syncmode full \
 --datadir /work/xdcchain --networkid 551 \
--port 30304 --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \
+-port 30303 --rpc --rpccorsdomain "*" --rpcaddr 0.0.0.0 \
 --rpcport 8545 \
 --rpcapi admin,db,eth,debug,miner,net,shh,txpool,personal,web3,XDPoS \
 --rpcvhosts "*" --unlock "${wallet}" --password /work/.pwd --mine \

cicd/devnet/terraform/.env

@@ -0,0 +1,2 @@
+num_of_nodes=2
+log_level=1

cicd/devnet/terraform/container-definition.tpl

@@ -3,7 +3,9 @@
     "name": "tfXdcNode",
     "image": "xinfinorg/${xdc_environment}:latest",
     "environment": [
-      {"name": "PRIVATE_KEYS", "value": "${private_keys}"}
+      {"name": "PRIVATE_KEYS", "value": "${private_keys}"},
+      {"name": "LOG_LEVEL", "value": "${log_level}"},
+      {"name": "NODE_NAME", "value": "${node_name}"}
     ],
     "essential": true,
     "logConfiguration": {
@@ -15,11 +17,6 @@
       }
     },
     "portMappings": [
-      {
-        "hostPort": 80,
-        "protocol": "tcp",
-        "containerPort": 80
-      },
       {
         "hostPort": 8555,
         "protocol": "tcp",
@@ -31,9 +28,9 @@
         "containerPort": 8545
       },
       {
-        "hostPort": 30304,
+        "hostPort": 30303,
         "protocol": "tcp",
-        "containerPort": 30304
+        "containerPort": 30303
       }
     ],
     "mountPoints": [

cicd/devnet/terraform/ecs.tf

@@ -1,16 +1,18 @@
 data template_file devnet_container_definition {
-  for_each = var.devnet_node_kyes
+  for_each = local.devnetNodeKyes
   template = "${file("${path.module}/container-definition.tpl")}"
 
   vars = {
     xdc_environment = "devnet"
-    private_keys = "${each.value.pk}",
+    node_name = "${each.key}"
+    private_keys = "${each.value.pk}"
     cloudwatch_group = "tf-${each.key}"
+    log_level = "${local.logLevel}"
   }
 }
 
 resource "aws_ecs_task_definition" "devnet_task_definition_group" {
-  for_each = var.devnet_node_kyes
+  for_each = local.devnetNodeKyes
   
   family = "devnet-${each.key}"
   requires_compatibilities = ["FARGATE"]
@@ -38,4 +40,44 @@ resource "aws_ecs_task_definition" "devnet_task_definition_group" {
   tags = {
        Name = "TfDevnetEcs-${each.key}"
    }
+}
+
+data "aws_ecs_task_definition" "devnet_ecs_task_definition" {
+  for_each = local.devnetNodeKyes
+  task_definition = aws_ecs_task_definition.devnet_task_definition_group[each.key].family
+}
+
+resource "aws_ecs_cluster" "devnet_ecs_cluster" {
+  name = "devnet-xdcnode-cluster"
+  tags = {
+    Name        = "TfDevnetEcsCluster"
+  }
+}
+
+resource "aws_ecs_service" "devnet_ecs_service" {
+  for_each = local.devnetNodeKyes
+  name                 = "ecs-service-${each.key}"
+  cluster              = aws_ecs_cluster.devnet_ecs_cluster.id
+  task_definition      = "${aws_ecs_task_definition.devnet_task_definition_group[each.key].family}:${max(aws_ecs_task_definition.devnet_task_definition_group[each.key].revision, data.aws_ecs_task_definition.devnet_ecs_task_definition[each.key].revision)}"
+  launch_type          = "FARGATE"
+  scheduling_strategy  = "REPLICA"
+  desired_count        = 1
+  force_new_deployment = true
+
+  network_configuration {
+    subnets          = [aws_subnet.devnet_subnet.id]
+    assign_public_ip = true
+    security_groups = [
+      aws_default_security_group.devnet_xdcnode_security_group.id
+    ]
+  }
+  
+  deployment_circuit_breaker {
+    enable = true
+    rollback = false
+  }
+
+  tags = {
+    Name        = "TfDevnetEcsService-${each.key}"
+  }
 }

cicd/devnet/terraform/efs.tf

@@ -1,5 +1,28 @@
 
 # EFS
+resource "aws_security_group" "devnet_efs_security_group" {
+  name = "TfDevnetEfsSecurityGroup"
+  description = "Allow HTTP in and out of devnet EFS"
+  vpc_id = aws_vpc.devnet_vpc.id
+
+  ingress {
+    from_port   = 2049
+    to_port     = 2049
+    protocol    = "TCP"
+    security_groups = [aws_default_security_group.devnet_xdcnode_security_group.id]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+  tags = {
+    Name = "TfDevnetEfs"
+  }
+}
+
 resource "aws_efs_file_system" "devnet_efs" {
    creation_token = "efs"
    performance_mode = "generalPurpose"
@@ -17,7 +40,7 @@ resource "aws_efs_mount_target" "devnet_efs_efs_mount_target" {
 }
 
 resource "aws_efs_access_point" "devnet_efs_access_point" {
-  for_each = var.devnet_node_kyes
+  for_each = local.devnetNodeKyes
   file_system_id = aws_efs_file_system.devnet_efs.id
   root_directory {
     path = "/${each.key}/database"

cicd/devnet/terraform/main.tf

@@ -13,24 +13,6 @@ provider "aws" {
   region  = "us-east-1"
 }
 
-# This bucket had to be created before you can run the terraform init
-resource "aws_s3_bucket" "terraform_s3_bucket" {
-  bucket = "terraform-devnet-bucket"
-  versioning {
-    enabled = true
-  }
-}
-
-# Bucket need to be created first. If first time run terraform init, need to comment out the below section
-terraform {
-  backend "s3" {
-    bucket = "terraform-devnet-bucket"
-    key    = "tf/terraform.tfstate"
-    region = "us-east-1"
-    encrypt = true
-  }
-}
-
 resource "aws_vpc" "devnet_vpc" {
   cidr_block = "10.0.0.0/16"
   instance_tenancy = "default"
@@ -99,31 +81,7 @@ resource "aws_default_security_group" "devnet_xdcnode_security_group" {
   }
 }
 
-resource "aws_security_group" "devnet_efs_security_group" {
-  name = "TfDevnetEfsSecurityGroup"
-  description = "Allow HTTP in and out of devnet EFS"
-  vpc_id = aws_vpc.devnet_vpc.id
-
-  ingress {
-    from_port   = 2049
-    to_port     = 2049
-    protocol    = "TCP"
-    security_groups = [aws_default_security_group.devnet_xdcnode_security_group.id]
-  }
-
-  egress {
-    from_port   = 0
-    to_port     = 0
-    protocol    = "-1"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-  tags = {
-    Name = "TfDevnetEfs"
-  }
-}
-
 # IAM policies
-
 data "aws_iam_policy_document" "xdc_ecs_tasks_execution_role" {
   statement {
     actions = ["sts:AssumeRole"]
@@ -154,7 +112,8 @@ resource "aws_iam_role_policy_attachment" "devnet_xdc_ecs_tasks_execution_role"
 
 # Logs
 resource "aws_cloudwatch_log_group" "devnet_cloud_watch_group" {
-  for_each = var.devnet_node_kyes
+  for_each = local.devnetNodeKyes
+
   name = "tf-${each.key}"
   retention_in_days = 14 # Logs are only kept for 14 days
   tags = {

cicd/devnet/terraform/s3.tf

@@ -0,0 +1,24 @@
+
+
+# This bucket had to be created before you can run the terraform init
+resource "aws_s3_bucket" "terraform_s3_bucket" {
+  bucket = "terraform-devnet-bucket"
+  versioning {
+    enabled = true
+  }
+}
+
+# Bucket need to be created first. If first time run terraform init, need to comment out the below section
+terraform {
+  backend "s3" {
+    bucket = "terraform-devnet-bucket"
+    key    = "tf/terraform.tfstate"
+    region = "us-east-1"
+    encrypt = true
+  }
+}
+
+data "aws_s3_bucket_object" "devnet_xdc_node_config" {
+  bucket = "terraform-devnet-bucket"
+  key    = "node-config.json"
+}

cicd/devnet/terraform/variables.tf

@@ -1,19 +1,18 @@
-variable "devnet_node_kyes" {
-  description = "Array of nodes keys."
-  type        = map(any)
-  
-  /**
-    Below is the list of private keys you need to specify. It follows the pattern of 
-    {{Name of the node}}: {
+locals {
+    /**
+    Load the nodes data from s3
+    Below is the the format the config needs to follow:
+    {{Name of the node, in a pattern of 'xdc'+ number. i.e xdc50}}: {
       pk: {{Value of the node private key}},
       ... any other configuration we want to pass.
     }
     Note: No `n` is allowed in the node name
   **/
-  default = {
-    xdc1 = {
-      pk  = "3efdb44088929167487da052125162b48d8d54fe8f7b7db11b5d5cc3b9a1c14b",
-      isChaosNode = false # This is a placeholder, config not supported yet
+    predefinedNodesConfig = jsondecode(data.aws_s3_bucket_object.devnet_xdc_node_config.body)
+    envs = { for tuple in regexall("(.*)=(.*)", file(".env")) : tuple[0] => tuple[1] }
+    logLevel = local.envs["log_level"]
+    keyNames =[for i in range(tonumber(local.envs["num_of_nodes"])) : "xdc${i}"]
+    devnetNodeKyes = {
+      for i in local.keyNames: i => local.predefinedNodesConfig[i]
     }
-  }
 }