I've created an example Next.js project with a GitLab CI/CD pipeline that builds a Docker image, pushes it to Amazon ECR, deploys it to an Amazon ECS Fargate cluster, and uploads static assets (JS, CSS, etc.) to Amazon S3. The example GitLab repo is here: https://gitlab.com/saltycrane/next-aws-ecr-ecs-gitlab-ci-cd-example

Interesting files¶

Here are the interesting parts of some of the files. See the full source code in the GitLab repo.

• .gitlab-ci.yml (view at gitlab)

  • the variables AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and ECR_HOST are set in the GitLab UI under "Settings" > "CI/CD" > "Variables"
  • this uses the saltycrane/aws-cli-and-docker Docker image which provides the aws v2 command line tools and docker in a single image. It is based on amazon/aws-cli and installs bc, curl, docker, jq, and tar. This idea is from Valentin's tutorial.

  variables:
    DOCKER_HOST: tcp://docker:2375
    DOCKER_TLS_CERTDIR: ""
    AWS_DEFAULT_REGION: "us-east-1"
    CI_APPLICATION_REPOSITORY: "$ECR_HOST/next-aws-ecr-ecs-gitlab-ci-cd-example"
    CI_APPLICATION_TAG: "$CI_PIPELINE_IID"
    CI_AWS_S3_BUCKET: "next-aws-ecr-ecs-gitlab-ci-cd-example"
    CI_AWS_ECS_CLUSTER: "next-aws-ecr-ecs-gitlab-ci-cd-example"
    CI_AWS_ECS_SERVICE: "next-aws-ecr-ecs-gitlab-ci-cd-example"
    CI_AWS_ECS_TASK_DEFINITION: "next-aws-ecr-ecs-gitlab-ci-cd-example"
    NEXT_JS_ASSET_URL: "https://$CI_AWS_S3_BUCKET.s3.amazonaws.com"
  
  stages:
    - build
    - deploy
  
  build:
    stage: build
    image: saltycrane/aws-cli-and-docker
    services:
      - docker:dind
    script:
      - ./bin/build-and-push-image-to-ecr
      - ./bin/upload-assets-to-s3
  
  deploy:
    stage: deploy
    image: saltycrane/aws-cli-and-docker
    services:
      - docker:dind
    script:
      - ./bin/ecs update-task-definition
  

• Dockerfile (view at gitlab)

  The value of NEXT_JS_ASSET_URL is passed in using the --build-arg option of the docker build command run in bin/build-and-push-image-to-ecr. It is used like an environment variable in the RUN npm run build command below. In this project it is assigned to assetPrefix in next.config.js.

  FROM node:14.16-alpine
  ARG NEXT_JS_ASSET_URL
  ENV NODE_ENV=production
  WORKDIR /app
  COPY ./package.json ./
  COPY ./package-lock.json ./
  RUN npm ci
  COPY . ./
  RUN npm run build
  EXPOSE 3000
  CMD ["npm", "start"]
  

• bin/build-and-push-image-to-ecr (view at gitlab)

  # log in to the amazon ecr docker registry
  aws ecr get-login-password | docker login --username AWS --password-stdin "$ECR_HOST"
  
  # build docker image
  docker pull "$CI_APPLICATION_REPOSITORY:latest" || true
  docker build --build-arg "NEXT_JS_ASSET_URL=$NEXT_JS_ASSET_URL" --cache-from "$CI_APPLICATION_REPOSITORY:latest" -t "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"     -t "$CI_APPLICATION_REPOSITORY:latest" .
  
  # push image to amazon ecr
  docker push "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG"
  docker push "$CI_APPLICATION_REPOSITORY:latest"
  

• bin/upload-assets-to-s3 (view at gitlab)

  LOCAL_ASSET_PATH=/tmp/upload-assets
  
  mkdir $LOCAL_ASSET_PATH
  
  # copy the generated assets out of the docker image
  docker run --rm --entrypoint tar "$CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG" cf - .next | tar xf - -C $LOCAL_ASSET_PATH
  
  # rename .next to _next
  mv "$LOCAL_ASSET_PATH/.next" "$LOCAL_ASSET_PATH/_next"
  
  # remove directories that should not be uploaded to S3
  rm -rf "$LOCAL_ASSET_PATH/_next/cache"
  rm -rf "$LOCAL_ASSET_PATH/_next/server"
  
  # gzip files
  find $LOCAL_ASSET_PATH -regex ".*\.\(css\|svg\|js\)$" -exec gzip {} \;
  
  # strip .gz extension off of gzipped files
  find $LOCAL_ASSET_PATH -name "*.gz" -exec sh -c 'mv $1 `echo $1 | sed "s/.gz$//"`' - {} \;
  
  # upload gzipped js, css, and svg assets
  aws s3 sync --no-progress $LOCAL_ASSET_PATH "s3://$CI_AWS_S3_BUCKET" --cache-control max-age=31536000 --content-encoding gzip --exclude "*" --include "*.js"     --include "*.css" --include "*.svg"
  
  # upload non-gzipped assets
  aws s3 sync --no-progress $LOCAL_ASSET_PATH "s3://$CI_AWS_S3_BUCKET" --cache-control max-age=31536000 --exclude "*.js" --exclude "*.css" --exclude "*.svg" --exclude "*.map"
  

• bin/ecs (view full file) (This file was copied from the gitlab-org repo)

  #!/bin/bash -e
  
  update_task_definition() {
    local -A register_task_def_args=( \
      ['task-role-arn']='taskRoleArn' \
      ['execution-role-arn']='executionRoleArn' \
      ['network-mode']='networkMode' \
      ['cpu']='cpu' \
      ['memory']='memory' \
      ['pid-mode']='pidMode' \
      ['ipc-mode']='ipcMode' \
      ['proxy-configuration']='proxyConfiguration' \
      ['volumes']='volumes' \
      ['placement-constraints']='placementConstraints' \
      ['requires-compatibilities']='requiresCompatibilities' \
      ['inference-accelerators']='inferenceAccelerators' \
    )
  
    image_repository=$CI_APPLICATION_REPOSITORY
    image_tag=$CI_APPLICATION_TAG
    new_image_name="${image_repository}:${image_tag}"
  
    register_task_definition_from_remote
  
    new_task_definition=$(aws ecs register-task-definition "${args[@]}")
    new_task_revision=$(read_task "$new_task_definition" 'revision')
    new_task_definition_family=$(read_task "$new_task_definition" 'family')
  
    # Making sure that we at least have one running task (even if desiredCount gets updated again with new task definition below)
    service_task_count=$(aws ecs describe-services --cluster "$CI_AWS_ECS_CLUSTER" --services "$CI_AWS_ECS_SERVICE" --query "services[0].desiredCount")
  
    if [[ $service_task_count == 0 ]]; then
      aws ecs update-service --cluster "$CI_AWS_ECS_CLUSTER" --service "$CI_AWS_ECS_SERVICE" --desired-count 1
    fi
  
    # Update ECS service with newly created task defintion revision.
    aws ecs update-service \
              --cluster "$CI_AWS_ECS_CLUSTER" \
              --service "$CI_AWS_ECS_SERVICE" \
              --task-definition "$new_task_definition_family":"$new_task_revision"
  
    return 0
  }
  
  read_task() {
    val=$(echo "$1" | jq -r ".taskDefinition.$2")
    if [ "$val" == "null" ];then
      val=$(echo "$1" | jq -r ".$2")
    fi
    if [ "$val" != "null" ];then
      echo -n "${val}"
    fi
  }
  
  register_task_definition_from_remote() {
    task=$(aws ecs describe-task-definition --task-definition "$CI_AWS_ECS_TASK_DEFINITION")
    current_container_definitions=$(read_task "$task" 'containerDefinitions')
    new_container_definitions=$(echo "$current_container_definitions" | jq --arg val "$new_image_name" '.[0].image = $val')
    args+=("--family" "${CI_AWS_ECS_TASK_DEFINITION}")
    args+=("--container-definitions" "${new_container_definitions}")
    for option in "${!register_task_def_args[@]}"; do
      value=$(read_task "$task" "${register_task_def_args[$option]}")
      if [ -n "$value" ];then
        args+=("--${option}" "${value}")
      fi
    done
  }
  
  update_task_definition
  

Usage - set up AWS resources¶

Below are the minimum steps I needed to create the required AWS services for my example. I use the AWS region "us-east-1". For info about creating some of these services via the command line, see my Amazon ECS notes.

Create an ECR repository

• create a private ECR repository here: https://console.aws.amazon.com/ecr/repositories?region=us-east-1
• name the repository "next-aws-ecr-ecs-gitlab-ci-cd-example"
• leave "Tag immutability" disabled to allow the "latest" tag to be overwritten

Create an ECS Fargate cluster

• click "Create Cluster" here: https://us-east-1.console.aws.amazon.com/ecs/home?region=us-east-1#/clusters
• choose "Networking only" (Fargate)
• name the cluster "next-aws-ecr-ecs-gitlab-ci-cd-example"
• check the "Create VPC" checkbox
• click "Create"

Create an ECS task definition

• click "Create new Task Definition" here: https://console.aws.amazon.com/ecs/home?region=us-east-1#/taskDefinitions
• select "FARGATE" and click "Next step"
• configure task
  • for "Task Definition Name" enter "next-aws-ecr-ecs-gitlab-ci-cd-example"
  • for "Task Role" select "None"
  • for "Task execution role" select "Create new role"
  • for "Task memory" select "0.5GB"
  • for "Task CPU" select "0.25 vCPU"
  • click "Add container"
    • for "Container Name" enter "next-aws-ecr-ecs-gitlab-ci-cd-example"
    • for "Image" enter "asdf" (this will be updated by the gitlab ci/cd job)
    • leave "Private repository authentication" unchecked
    • for "Port mappings" enter "3000"
    • click "Add"
  • click "Create"

Create an ECS service

• click "Create" here: https://us-east-1.console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/next-aws-ecr-ecs-gitlab-ci-cd-example/services
• configure service
  • for "Launch type" select "FARGATE"
  • for "Task Definition" enter "next-aws-ecr-ecs-gitlab-ci-cd-example"
  • for "Cluster" select "next-aws-ecr-ecs-gitlab-ci-cd-example"
  • for "Service name" enter "next-aws-ecr-ecs-gitlab-ci-cd-example"
  • for "Number of tasks" enter 1
  • for "Deployment type" select "Rolling update"
  • click "Next step"
• configure network
  • select the appropriate "Cluster VPC" and two "Subnets"
  • click "Next step"
• set Auto Scaling
  • click "Next step"
• review
  • click "Create Service"

Open port 3000

• on the ECS service page https://us-east-1.console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/next-aws-ecr-ecs-gitlab-ci-cd-example/services/next-aws-ecr-ecs-gitlab-ci-cd-example/details under "Network Access", next to "Security groups", click the link to the security group
• click "Actions" then click "Edit inbound rules"
• click "Add rule"
• for "Port range" enter "3000"
• for "Source" select "0.0.0.0/0"
• click "Save rules"

Create a S3 bucket

• click "Create bucket" here: https://s3.console.aws.amazon.com/s3/home?region=us-east-1
• for "Bucket name" enter "next-aws-ecr-ecs-gitlab-ci-cd-example"
• uncheck "Block all public access"
• check the "I acknowledge that the current settings might result in this bucket and the objects within becoming public" checkbox
• click "Create bucket"

Update permissions for S3 bucket

• go to Permissions (https://s3.console.aws.amazon.com/s3/buckets/next-aws-ecr-ecs-gitlab-ci-cd-example?region=us-east-1&tab=permissions) and under "Bucket policy", click "Edit"
• enter:

  {
    "Version": "2012-10-17",
    "Statement": [
      {
        "Effect": "Allow",
        "Principal": "*",
        "Action": "s3:GetObject",
        "Resource": "arn:aws:s3:::next-aws-ecr-ecs-gitlab-ci-cd-example/*"
      }
    ]
  }
  

• click "Save changes"

Create an IAM user

• create an IAM user. The user must have at least ECR, ECS, and S3 permissions.
• take note of the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY

Usage - run the CI/CD pipeline¶

Fork the example gitlab repo and configure CI/CD variables

• fork https://gitlab.com/saltycrane/next-aws-ecr-ecs-gitlab-ci-cd-example
• go to "Settings" > "CI/CD" > "Variables" and add the following variables. You can choose to "protect" and "mask" all of them.
  • AWS_ACCESS_KEY_ID
  • AWS_SECRET_ACCESS_KEY
  • ECR_HOST (This is the part of the ECR repository URI before the /. It looks something like XXXXXXXXXXXX.dkr.ecr.us-east-1.amazonaws.com)

Edit variables in .gitlab-ci.yml

If you used names other than "next-aws-ecr-ecs-gitlab-ci-cd-example", edit the variables in .gitlab-ci.yml.

Test it

• clone the repo and push a commit
• see the pipeline running under "CI/CD" > "Pipelines"
• go to the cluster tasks page: https://us-east-1.console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/next-aws-ecr-ecs-gitlab-ci-cd-example/tasks
• click on the task and copy the "Public IP"
• enter the public IP followed by :3000 in the browser (Note: the IP address changes for every git push. A load balancer should probably be used, but I didn't do that.)

References (build/push)¶

• https://www.youtube.com/watch?v=jg9sUceyGaQ
• https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#docker
• https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#using-docker-caching

References (deploy)¶

• https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs
• https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml
• https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
• https://gitlab.com/gitlab-org/cloud-deploy/-/blob/master/aws/ecs/Dockerfile
• https://gitlab.com/gitlab-org/cloud-deploy/-/blob/master/aws/src/bin/ecs

These are my notes for creating "hello world" and "docker hello world" GitLab CI/CD pipelines.

GitLab CI/CD hello world (gitlab repo) ¶

• create git repo, push it to gitlab, and set origin. Replace saltycrane with your username.

  mkdir gitlab-ci-cd-hello-world
  cd gitlab-ci-cd-hello-world
  git init
  touch .gitignore
  git add .
  git commit -m 'first commit'
  git push --set-upstream [email protected]:saltycrane/gitlab-ci-cd-hello-world.git --all
  git remote add origin [email protected]:saltycrane/gitlab-ci-cd-hello-world.git
  

• add a .gitlab-ci.yml file

  build-hello:
    script:
      - echo "hello world"
  

• commit and push

  git add .
  git commit -m 'add ci/cd config'
  git push origin
  

• see the pipeline run (replace saltycrane with your username): https://gitlab.com/saltycrane/gitlab-ci-cd-hello-world/-/pipelines

GitLab CI/CD Docker hello world (gitlab repo) ¶

• create git repo, push it to gitlab, and set origin. Replace saltycrane with your username.

  mkdir gitlab-ci-cd-docker-hello-world
  cd gitlab-ci-cd-docker-hello-world
  git init
  touch .gitignore
  git add .
  git commit -m 'first commit'
  git push --set-upstream [email protected]:saltycrane/gitlab-ci-cd-docker-hello-world.git --all
  git remote add origin [email protected]:saltycrane/gitlab-ci-cd-docker-hello-world.git
  

• add a Dockerfile file

  FROM alpine
  RUN echo "hello"
  

• add a .gitlab-ci.yml file

  variables:
    DOCKER_TLS_CERTDIR: "/certs"
  
  build-docker:
    image: docker:latest
    services:
      - docker:dind
    script:
      - docker build -t hello .
  

• commit and push

  git add .
  git commit -m 'add Dockerfile and ci/cd config'
  git push origin
  

• see the pipeline run (replace saltycrane with your username): https://gitlab.com/saltycrane/gitlab-ci-cd-docker-hello-world/-/pipelines

These are my notes for creating a Docker image, pushing it to Amazon ECR (Elastic Container Registry), and deploying it to Amazon ECS (Elastic Container Service) using AWS Fargate (serverless for containers) using command line tools.

Create docker image on local machine ¶

• install docker (macOS)

  brew install homebrew/cask/docker
  

• create directory

  mkdir /tmp/my-project
  cd /tmp/my-project
  

• create /tmp/my-project/Dockerfile:

  FROM python:3.9-alpine3.13
  WORKDIR /app
  RUN echo 'Hello' > ./index.html
  EXPOSE 80
  CMD ["python", "-m", "http.server", "80"]
  

• create Docker image

  docker build -t my-image .
  

• test running the Docker image locally

  docker run -p 80:80 my-image
  

• go to http://localhost in the browser and see the text "Hello"

Install and configure AWS command line tools ¶

• install AWS command line tools

  brew install awscli
  

• create an IAM user

• run aws configure and enter:

  • AWS Access Key ID
  • AWS Secret Access Key

  This creates the file ~/.aws/credentials

Create ECR repository and push image to it ¶

From https://docs.aws.amazon.com/AmazonECS/latest/developerguide/docker-basics.html#use-ecr

• create an Amazon ECR repository using aws ecr create-repository

  aws ecr create-repository --repository-name my-repository --region us-east-1
  

  output:

  {
      "repository": {
          "repositoryArn": "arn:aws:ecr:us-east-1:AAAAAAAAAAAA:repository/my-repository",
          "registryId": "AAAAAAAAAAAA",
          "repositoryName": "my-repository",
          "repositoryUri": "AAAAAAAAAAAA.dkr.ecr.us-east-1.amazonaws.com/my-repository",
          "createdAt": "2021-03-17T10:48:18-07:00",
          "imageTagMutability": "MUTABLE",
          "imageScanningConfiguration": {
              "scanOnPush": false
          },
          "encryptionConfiguration": {
              "encryptionType": "AES256"
          }
      }
  }
  

  Take note of the "registryId" and use it in place of "AAAAAAAAAAAA" below.

• tag the docker image with the repositoryUri

  docker tag my-image AAAAAAAAAAAA.dkr.ecr.us-east-1.amazonaws.com/my-repository
  

• log in to the Amazon ECR registry using aws ecr get-login-password

  aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin AAAAAAAAAAAA.dkr.ecr.us-east-1.amazonaws.com
  

• push the docker image to the Amazon ECR repository

  docker push AAAAAAAAAAAA.dkr.ecr.us-east-1.amazonaws.com/my-repository
  

• see the image in AWS console https://console.aws.amazon.com/ecr/repositories?region=us-east-1

Install ECS command line tools¶

• install ecs-cli. Note there is ecs-cli in addition to aws ecs tools. The reason is probably similar to why some services are named "Amazon Service" and some are named "AWS Service". (It seems like ecs-cli provides higher level commands.)

  brew install amazon-ecs-cli
  

Create Amazon ECS Fargate cluster ¶

From https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-cli-tutorial-fargate.html

• create a cluster using ecs-cli up

  ecs-cli up --cluster my-cluster --launch-type FARGATE --region us-east-1
  

  output:

  INFO[0001] Created cluster                               cluster=my-cluster region=us-east-1
  INFO[0002] Waiting for your cluster resources to be created...
  INFO[0002] Cloudformation stack status                   stackStatus=CREATE_IN_PROGRESS
  VPC created: vpc-BBBBBBBBBBBBBBBBB
  Subnet created: subnet-CCCCCCCCCCCCCCCCC
  Subnet created: subnet-DDDDDDDDDDDDDDDDD
  Cluster creation succeeded.
  

  Take note of the VPC (virtual private cloud), and two subnet IDs to use later. See the cluster in the AWS console UI: https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters

Gather parameters required to deploy to ECS cluster ¶

From https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-cli-tutorial-fargate.html

Create task execution IAM role¶

• create a file /tmp/my-project/task-execution-assume-role.json

  {
    "Version": "2012-10-17",
    "Statement": [
      {
        "Sid": "",
        "Effect": "Allow",
        "Principal": {
          "Service": "ecs-tasks.amazonaws.com"
        },
        "Action": "sts:AssumeRole"
      }
    ]
  }
  

• create the task execution role using aws iam create-role

  aws iam create-role --role-name my-task-execution-role --assume-role-policy-document file:///tmp/my-project/task-execution-assume-role.json --region us-east-1
  

• attach the task execution role policy using aws iam attach-role-policy

  aws iam attach-role-policy --role-name my-task-execution-role --policy-arn arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy --region us-east-1
  

Get security group ID¶

• get the default security group ID for the virtual private cloud (VPC) created when creating the ECS cluster using aws ec2 describe-security-groups. Replace "vpc-BBBBBBBBBBBBBBBBB" with your VPC ID

  aws ec2 describe-security-groups --filters Name=vpc-id,Values=vpc-BBBBBBBBBBBBBBBBB --region us-east-1
  

  output:

  {
      "SecurityGroups": [
          {
              "Description": "default VPC security group",
              "GroupName": "default",
              "IpPermissions": [
                  {
                      "IpProtocol": "-1",
                      "IpRanges": [],
                      "Ipv6Ranges": [],
                      "PrefixListIds": [],
                      "UserIdGroupPairs": [
                          {
                              "GroupId": "sg-EEEEEEEEEEEEEEEEE",
                              "UserId": "AAAAAAAAAAAA"
                          }
                      ]
                  }
              ],
              "OwnerId": "AAAAAAAAAAAA",
              "GroupId": "sg-EEEEEEEEEEEEEEEEE",
              "IpPermissionsEgress": [
                  {
                      "IpProtocol": "-1",
                      "IpRanges": [
                          {
                              "CidrIp": "0.0.0.0/0"
                          }
                      ],
                      "Ipv6Ranges": [],
                      "PrefixListIds": [],
                      "UserIdGroupPairs": []
                  }
              ],
              "VpcId": "vpc-BBBBBBBBBBBBBBBBB"
          }
      ]
  }
  

  Take note of the "GroupId" to be used later

Deploy to Amazon ECS cluster¶

From https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-cli-tutorial-fargate.html

• create /tmp/my-project/ecs-params.yml replacing "subnet-CCCCCCCCCCCCCCCCC", "subnet-DDDDDDDDDDDDDDDDD", and "sg-EEEEEEEEEEEEEEEEE" with appropriate IDs from above. ECS Parameters docs

  version: 1
  task_definition:
    task_execution_role: my-task-execution-role
    ecs_network_mode: awsvpc
    task_size:
      mem_limit: 0.5GB
      cpu_limit: 256
  run_params:
    network_configuration:
      awsvpc_configuration:
        subnets:
          - "subnet-CCCCCCCCCCCCCCCCC"
          - "subnet-DDDDDDDDDDDDDDDDD"
        security_groups:
          - "sg-EEEEEEEEEEEEEEEEE"
        assign_public_ip: ENABLED
  

• create /tmp/my-project/docker-compose.yml replacing AAAAAAAAAAAA with the registryId:

  version: '3'
  services:
    web:
      image: 'AAAAAAAAAAAA.dkr.ecr.us-east-1.amazonaws.com/my-repository'
      ports:
        - '80:80'
  

• deploy to the ECS cluster using ecs-cli compose service up. This creates a task definition and service. This uses the docker-compose.yml file in the current directory.

  ecs-cli compose --cluster my-cluster --project-name my-project --ecs-params ecs-params.yml --region us-east-1 service up --launch-type FARGATE
  

  see the service in the web UI: https://console.aws.amazon.com/ecs/home?region=us-east-1#/clusters/my-cluster/services

Hit the server in the browser¶

• configure security group to allow inbound access on port 80 using aws ec2 authorize-security-group-ingress

  aws ec2 authorize-security-group-ingress --group-id sg-EEEEEEEEEEEEEEEEE --protocol tcp --port 80 --cidr 0.0.0.0/0 --region us-east-1
  

• get the IP address using ecs-cli compose service ps

  ecs-cli compose --cluster my-cluster --project-name my-project --region us-east-1 service ps
  

  output:

  Name                                             State    Ports                    TaskDefinition  Health
  my-cluster/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx/web  RUNNING  FF.FF.FF.FF:80->80/tcp  my-project:1    UNKNOWN
  

  Take note of the IP address under "Ports"

• visit in the browser: http://FF.FF.FF.FF replacing "FF.FF.FF.FF" with your IP address

Destroy¶

• delete the ECS service using ecs-cli compose service down

  ecs-cli compose --cluster my-cluster --project-name my-project --region us-east-1 service down
  

• delete the ECS cluster using ecs-cli down

  ecs-cli down --force --cluster my-cluster --region us-east-1
  

• delete the ECR repository using aws ecr delete-repository

  aws ecr delete-repository --repository-name my-repository --region us-east-1 --force
  

References¶

• https://docs.aws.amazon.com/AmazonECS/latest/developerguide/docker-basics.html
• https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-cli-tutorial-fargate.html

This is a TypeScript Node.js script to download GitHub pull request information (title, body, comments, etc.) using the GitHub GraphQL API. The data is saved in a JSON file.

The GitHub repo is here: download-github-prs.

Create a GitHub personal access token

Create a GitHub personal access token as described here (no checkboxes need to be selected for public repos): https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token This is used to access the GitHub GraphQL API.

Intall libraries

• Create a directory and cd into it

  $ mkdir -p /tmp/my-prs
  $ cd /tmp/my-prs 
  

• Create package.json file:

  {
    "scripts": {
      "download": "ts-node index.ts"
    },
    "dependencies": {
      "node-fetch": "^2.6.1"
    },
    "devDependencies": {
      "@types/node-fetch": "^2.5.7",
      "ts-node": "^9.0.0",
      "typescript": "^4.1.2"
    }
  }
  

• Install

  $ npm install 
  

Create script

Creat a file, /tmp/my-prs/index.ts, replacing XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX with the GitHub personal access token described above.

import * as fs from "fs";
import fetch from "node-fetch";

const GITHUB_TOKEN = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
const GITHUB_GRAPHQL_API_URL = "https://api.github.com/graphql";
const OUTPUT_DIR = "/tmp/my-prs";
const REPO_OWNER = "facebookexperimental";
const REPO_NAME = "Recoil";

fetchPullRequest(1);

/**
 * fetchPullRequest
 */
async function fetchPullRequest(prNumber: number) {
  const reactionFragment = `
    content
    user {
      login
    }
  `;
  const userContentEditFragment = `
    createdAt
    deletedAt
    deletedBy {
      login
    }
    diff
    editedAt
    editor {
      login
    }
    updatedAt
  `;
  const commentFragment = `
    author {
      login
    }
    body
    createdAt
    reactions(first: 100) {
      nodes {
        ${reactionFragment}
      }
    }
    userContentEdits(first: 100) {
      nodes {
        ${userContentEditFragment}
      }
    }
  `;
  const query = `
  query {
    repository(owner: "${REPO_OWNER}", name: "${REPO_NAME}") {
      nameWithOwner
      pullRequest(number: ${prNumber}) {
        author { login }
        baseRefName
        baseRefOid
        body
        closedAt
        comments(first: 100) {
          nodes {
            ${commentFragment}
          }
        }
        commits(first: 250) {
          nodes {
            commit {
              oid
            }
          }
        }
        createdAt
        files(first: 100) {
          nodes { path }
        }
        headRefName
        headRefOid
        mergeCommit { oid }
        merged
        mergedAt
        mergedBy { login }
        number
        publishedAt
        reactions(first: 10) {
          nodes {
            ${reactionFragment}
          }
        }
        reviews(first: 10) {
          nodes {
            author { login }
            body
            comments(first: 10) {
              nodes {
                ${commentFragment}
              }
            }
            commit {
              oid
            }
            createdAt
            editor { login }
            publishedAt
            reactions(first: 10) {
              nodes {
                ${reactionFragment}
              }
            }
            resourcePath
            submittedAt
            updatedAt
            userContentEdits(first: 10) {
              nodes {
                ${userContentEditFragment}
              }
            }
          }
        }
        state
        title
        updatedAt
        userContentEdits(first: 10) {
          nodes {
            ${userContentEditFragment}
          }
        }
      }
    }
  }
  `;

  // make graphql query and strigify the response
  const resp = await fetchQuery(query);
  const respStr = JSON.stringify(resp, null, 2);

  // save json file
  const filepath = [
    `${OUTPUT_DIR}/`,
    `${REPO_NAME}-pr-${String(prNumber).padStart(4, "0")}.json`,
  ].join("");
  console.log(`Saving ${filepath}...`);
  fs.writeFileSync(filepath, respStr);
}

/**
 * fetchQuery
 */
function fetchQuery(query: string, variables: Record<string, any> = {}) {
  return fetch(GITHUB_GRAPHQL_API_URL, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      Authorization: `bearer ${GITHUB_TOKEN}`,
    },
    body: JSON.stringify({
      query,
      variables,
    }),
  }).then((response) => {
    return response.json();
  });
}

Run it

$ npm run download 

Output

This produces the following JSON file

$ cat /tmp/my-prs/Recoil-pr-0001.json
{
  "data": {
    "repository": {
      "nameWithOwner": "facebookexperimental/Recoil",
      "pullRequest": {
        "author": {
          "login": "facebook-github-bot"
        },
        "baseRefName": "master",
        "baseRefOid": "40e870caadc159a87e81be291ff641410ab32e8f",
        "body": "This is pull request was created automatically because we noticed your project was missing a Contributing file.\n\nCONTRIBUTING files explain how a developer can contribute to the project - which you should actively encourage.\n\nThis PR was crafted with love by Facebook's Open Source Team.",
        "closedAt": "2020-05-13T04:12:15Z",
        "comments": {
          "nodes": [
            {
              "author": {
                "login": "davidmccabe"
              },
              "body": "Already added this manually.",
              "createdAt": "2020-05-13T04:12:15Z",
              "reactions": {
                "nodes": []
              },
              "userContentEdits": {
                "nodes": []
              }
            }
          ]
        },
        "commits": {
          "nodes": [
            {
              "commit": {
                "oid": "96f91679540362fa96a6c92611a8ef5621447b42"
              }
            }
          ]
        },
        "createdAt": "2020-05-06T22:31:01Z",
        "files": {
          "nodes": [
            {
              "path": "CONTRIBUTING.md"
            }
          ]
        },
        "headRefName": "automated_fixup_contributing_file_exists",
        "headRefOid": "96f91679540362fa96a6c92611a8ef5621447b42",
        "mergeCommit": null,
        "merged": false,
        "mergedAt": null,
        "mergedBy": null,
        "number": 1,
        "publishedAt": "2020-05-06T22:31:01Z",
        "reactions": {
          "nodes": []
        },
        "reviews": {
          "nodes": []
        },
        "state": "CLOSED",
        "title": "Adding Contributing file",
        "updatedAt": "2020-10-07T20:23:05Z",
        "userContentEdits": {
          "nodes": []
        }
      }
    }
  }
}

Fabric 2 is a Python package used for running commands on remote machines via SSH. Fabric 2 supports Python 3 and is a rewrite of the Fabric I used years ago. Here are my notes on using Fabric 2 and Python 3.

Set up SSH config and SSH agent

• Create or edit your ~/.ssh/config file to contain your remote host parameters

  Host myhost
      User myusername
      HostName myhost.com
      IdentityFile ~/.ssh/id_rsa
  

• Add your private key to your SSH agent

  $ ssh-add ~/.ssh/id_rsa
  

Create a project, create a virtualenv, and install fabric2

$ mkdir -p /tmp/my-project
$ cd /tmp/my-project
$ python3 -m venv venv
$ source venv/bin/activate
$ pip install fabric2

Create a fabfile.py script

Create a file /tmp/my-project/fabfile.py with the following contents. Note: "myhost" is the same name used in ~/.ssh/config described above.

from fabric2 import task

hosts = ["myhost"]

@task(hosts=hosts)
def mytask(c):
    print("Starting mytask...")
    with c.cd("/var"):
        c.run("ls -l")
    print("Done.")

Run the fabric script

In /tmp/my-project, with the virtualenv activated, run the fabric task to list the contents of /var on the remote host.

$ fab2 mytask 

Output:

Starting mytask...

total 48
drwxr-xr-x  2 root root   4096 backups
drwxr-xr-x  9 root root   4096 cache
drwxrwxrwt  2 root root   4096 crash
drwxr-xr-x 38 root root   4096 lib
drwxrwsr-x  2 root root   4096 local
drwxrwxrwt  2 root root   4096 lock
drwxrwxr-x 14 root root   4096 log
drwxrwsr-x  2 root root   4096 mail
drwxr-xr-x  2 root root   4096 opt
drwxr-xr-x  5 root root   4096 spool
drwxrwxrwt  2 root root   4096 tmp
drwxr-xr-x  3 root root   4096 www

Done.

See also / References

• Example fabfile (Upgrading from 1.x docs)
• paramiko SSHClient API reference

This describes an example GraphQL API using Relay connections to power a discretely paged Amazon-like pagination UI. Connections are designed for infinite-scroll UIs, however Artsy Engineering developed a solution to use them with discretely paged UIs as described in their article: Effortless Pagination with GraphQL and Relay? Really!

I created a github repo with an implementation of the Artsy pagination API in Graphene, Django, and Python and a corresponding web app using Relay, React, and TypeScript.

For a vanilla Graphene Relay pagination API for an infinite scroll UI, see my basic example.

Graphene GraphQL backend¶

The Artsy pagination API augments the Relay Connection type with an additional pageCursors field that contains metadata used to implement the pagination UI. I ported their TypeScript code in artsy/metaphysics and relay-cursor-paging to Python using Graphene and Django.

Data is stored in a PostgreSQL database running in Docker. It is seeded with fish fixture data from ACNHAPI. Django is configured to log SQL queries to the console and show that SQL queries use COUNT, LIMIT, and OFFSET. Example:

SELECT COUNT(*) AS "__count" FROM "fishes_fish";
SELECT "fishes_fish"."id", "fishes_fish"."description", "fishes_fish"."icon_url", "fishes_fish"."name", "fishes_fish"."price" FROM "fishes_fish" ORDER BY "fishes_fish"."name" ASC LIMIT 5 OFFSET 5;

• Install Python 3.8

• Install packages, set up database, and run dev server

  $ git clone https://github.com/saltycrane/graphene-relay-pagination-example.git
  $ cd graphene-relay-pagination-example
  $ cd graphene-api
  $ python3 -m venv venv
  $ source venv/bin/activate
  $ pip install -r requirements.txt
  $ docker-compose up -d
  $ ./bin/resetdb
  $ ./manage.py migrate
  $ ./manage.py loaddata fishes
  $ ./manage.py runserver
  

• Go to http://127.0.0.1:8000/graphql/ in the browser

• Run the following query:

  {
    allFishes(first: 5, orderBy: "name") {
      pageCursors {
        previous {
          cursor
        }
        first {
          cursor
          page
        }
        around {
          cursor
          isCurrent
          page
        }
        last {
          cursor
          page
        }
        next {
          cursor
        }
      }
      edges {
        cursor
        node {
          name
        }
      }
    }
  }

  GraphiQL query screenshot
  

$ ./manage.py graphql_schema --schema pagination_ex_api.schema.schema --out ../schema.graphql

React Relay Next.js frontend¶

In addition to Relay, React, and TypeScript, the frontend UI uses relay-hooks, Next.js, and reactstrap. It takes advantage of Relay fragments and Next.js routing to store pagination state. Server-side rendering (SSR) is disabled because it's difficult to set up and isn't important for this example.

• Install Node.js 14

• Install packages and run dev server

  $ cd react-relay-webapp
  $ npm install
  $ npm run devserver
  

• Go to http://127.0.0.1:3000 in the browser

Links to code¶

• /graphene-api/fishes/schema.py
• /graphene-api/artsy_relay_pagination/fields.py
• /graphene-api/artsy_relay_pagination/pagination.py

• /react-relay-webapp/src/FishesPage.tsx
• /react-relay-webapp/src/FishesPagination.tsx

• https://github.com/artsy/metaphysics/blob/production--2021-01-22--15-05-38/src/schema/v2/fields/pagination.ts
• https://github.com/artsy/metaphysics/blob/production--2021-01-22--15-05-38/src/lib/helpers.ts#L90-L104
• https://github.com/darthtrevino/relay-cursor-paging/blob/v0.2.0/src/getPagingParameters.ts
• https://github.com/graphql/graphql-relay-js/blob/v0.5.4/src/connection/arrayconnection.js

• https://github.com/graphql-python/graphene-django/blob/v2.15.0/graphene_django/filter/fields.py
• https://github.com/graphql-python/graphene-django/blob/v2.15.0/graphene_django/fields.py#L132-L177
• https://github.com/graphql-python/graphql-relay-py/blob/v2.0.1/graphql_relay/connection/arrayconnection.py#L30-L104

React is used to build web applications that run JavaScript in a user's browser (client side rendering). It can also be used from a Node.js script to generate static HTML (static rendering). I used this technique to generate some CSS width experiments and a TypeScript Next.js cheat sheet. The example below shows how to use React with TypeScript and Node.js to generate static HTML. I also made an example project on github.

Install Node.js¶

• $ brew install node

Set up project¶

• Create a project directory and cd into it

  $ mkdir my-project
  $ cd my-project

• Create a package.json file:

  {
    "scripts": {
      "render": "tsc && node dist/render.js"
    },
    "dependencies": {
      "@types/node": "^14.0.4",
      "@types/prettier": "^2.0.0",
      "@types/react": "^16.9.35",
      "@types/react-dom": "^16.9.8",
      "prettier": "^2.0.5",
      "react": "^16.13.1",
      "react-dom": "^16.13.1",
      "typescript": "^3.9.3"
    }
  }

• Install React, TypeScript, and other packages

  $ npm install

• Create a tsconfig.json file to configure TypeScript

  {
    "compilerOptions": {
      "baseUrl": ".",
      "esModuleInterop": true,
      "jsx": "react",
      "lib": ["dom", "es2019"],
      "outDir": "dist",
      "paths": {
        "*": ["src/*", "src/@types/*"]
      }
    },
    "include": ["src/*.tsx"]
  }

Create a script to generate a static HTML file¶

• Create a directory, src and a file src/render.tsx:

  import * as fs from "fs";
  import prettier from "prettier";
  import React from "react";
  import ReactDOMServer from "react-dom/server";
  
  render();
  
  function render() {
    let html = ReactDOMServer.renderToStaticMarkup(<HelloWorldPage />);
    let htmlWDoc = "<!DOCTYPE html>" + html;
    let prettyHtml = prettier.format(htmlWDoc, { parser: "html" });
    let outputFile = "./output.html";
    fs.writeFileSync(outputFile, prettyHtml);
    console.log(`Wrote ${outputFile}`);
  }
  
  function HelloWorldPage() {
    return (
      <html lang="en">
        <head>
          <meta charSet="utf-8" />
          <title>Hello world</title>
        </head>
        <body>
          <h1>Hello world</h1>
        </body>
      </html>
    );
  }

Run the script and view the output¶

• Run the script

  $ npm run render

• Open the output.html file in the browser

  $ open output.html