feat: Automate Layer Releases with R2R artifacts

.gitignore

@@ -1,3 +1,5 @@
 .layers
 .DS_Store
 Datadog.Trace.dll
+
+.gitlab/build-pipeline.yaml

.gitlab-ci.yml

@@ -0,0 +1,85 @@
+variables:
+  DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/dd-trace-dotnet-aws-lambda-layer
+  DOCKER_TARGET_VERSION: latest
+  # Default version for development builds
+  # This will be overwritten by the tag version if it is a release.
+  VERSION: dev
+  # Manual trigger variables
+  TRACER_BRANCH:
+    description: "Branch of the dd-trace-dotnet repository to use (default empty)."
+    value: ""
+  TRACER_VERSION:
+    description: "Latest release version of the dd-trace-dotnet to tag the build with (default empty)."
+    value: ""
+  LAYER_SUFFIX:
+    description: "Suffix to be appended to the layer name (default empty)."
+    value: ""
+
+stages:
+  - pre 
+  - build
+
+.go-cache: &go-cache
+  key: dd-trace-dotnet-aws-lambda-layer-go-cache
+  policy: pull
+
+ci image:
+  stage: build
+  image: registry.ddbuild.io/images/docker:20.10
+  tags: ["arch:arm64"]
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"'
+      changes:
+        - .gitlab/Dockerfile
+      when: on_success
+  variables:
+    DOCKER_TARGET: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION}
+  script:
+    - docker buildx build --platform linux/amd64,linux/arm64 --no-cache --pull --push --tag ${DOCKER_TARGET} -f .gitlab/Dockerfile .
+
+generator:
+  stage: pre
+  image: registry.ddbuild.io/images/mirror/golang:alpine
+  tags: ["arch:amd64"]
+  cache: *go-cache
+  rules:
+    - if: '$CI_PIPELINE_SOURCE =~ /external_pull_request_event|merge_request_event|push/'
+      when: never
+    - when: always
+  script:
+    - if [[ "$CI_COMMIT_TAG" =~ ^v[0-9]+$ ]]; then echo "VERSION=${CI_COMMIT_TAG//[!0-9]/}" >> .env; fi
+    # These variables are used to trigger the build job manually, when the pipeline is triggered by an external source
+    # these are not set, so we reset them to a placeholder.
+    - if [ -z "$TRACER_BRANCH" ] || [ "$TRACER_BRANCH" = "placeholder" ]; then echo "TRACER_BRANCH=placeholder" >> .env; fi
+    - if [ -z "$TRACER_VERSION" ] || [ "$TRACER_VERSION" = "placeholder" ]; then echo "TRACER_VERSION=placeholder" >> .env; fi
+    - if [ -z "$UPSTREAM_PIPELINE_ID" ] || [ "$UPSTREAM_PIPELINE_ID" = "placeholder" ]; then echo "UPSTREAM_PIPELINE_ID=placeholder" >> .env; fi
+    - if [ -z "$LAYER_SUFFIX" ] || [ "$LAYER_SUFFIX" = "placeholder" ]; then echo "LAYER_SUFFIX=placeholder" >> .env; fi
+    - apk add --no-cache gomplate
+    - gomplate --config .gitlab/config.yaml
+  artifacts:
+    paths:
+      - .gitlab/build-pipeline.yaml
+    reports:
+      dotenv: .env
+
+build:
+  stage: build
+  needs: ["generator"]
+  trigger:
+    include:
+      - artifact: .gitlab/build-pipeline.yaml
+        job: generator
+    strategy: depend
+  needs:
+    - job: generator
+      artifacts: true
+  rules:
+    - if: '$CI_PIPELINE_SOURCE =~ /external_pull_request_event|merge_request_event|push/'
+      when: never
+    - when: always
+  variables:
+    UPSTREAM_PIPELINE_ID: $UPSTREAM_PIPELINE_ID
+    VERSION: $VERSION
+    TRACER_BRANCH: $TRACER_BRANCH
+    LAYER_SUFFIX: $LAYER_SUFFIX
+    TRACER_VERSION: $TRACER_VERSION

.gitlab/Dockerfile

@@ -0,0 +1,8 @@
+FROM registry.ddbuild.io/images/docker:24.0.5
+
+RUN apt-get update && apt-get install -y --fix-missing --no-install-recommends \
+  curl gcc gnupg g++ make cmake unzip openssl g++ uuid-runtime
+
+# Install AWS CLI
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip && ./aws/install

.gitlab/config.yaml

@@ -0,0 +1,17 @@
+# gomplate template generation pipeline
+
+inputFiles:
+  - .gitlab/template.yaml.tpl
+
+outputFiles:
+  - .gitlab/build-pipeline.yaml
+
+datasources:
+  architectures:
+    url: .gitlab/datasources/architectures.yaml
+
+  environments:
+    url: .gitlab/datasources/environments.yaml
+
+  regions:
+    url: .gitlab/datasources/regions.yaml

.gitlab/datasources/architectures.yaml

@@ -0,0 +1,3 @@
+architectures:
+  - name: amd64
+  - name: arm64

.gitlab/datasources/environments.yaml

@@ -0,0 +1,9 @@
+environments:
+  - name: sandbox
+    external_id: sandbox-publish-externalid
+    role_to_assume: sandbox-layer-deployer
+    account: 425362996713
+  - name: prod
+    external_id: prod-publish-externalid
+    role_to_assume: dd-serverless-layer-deployer-role
+    account: 464622532012

.gitlab/datasources/regions.yaml

@@ -0,0 +1,30 @@
+regions:
+  - code: "us-east-1"
+  - code: "us-east-2"
+  - code: "us-west-1"
+  - code: "us-west-2"
+  - code: "af-south-1"
+  - code: "ap-east-1"
+  - code: "ap-south-1"
+  - code: "ap-south-2"
+  - code: "ap-southeast-1"
+  - code: "ap-southeast-2"
+  - code: "ap-southeast-3"
+  - code: "ap-southeast-4"
+  - code: "ap-northeast-1"
+  - code: "ap-northeast-2"
+  - code: "ap-northeast-3"
+  - code: "ca-central-1"
+  - code: "ca-west-1"
+  - code: "eu-central-1"
+  - code: "eu-central-2"
+  - code: "eu-north-1"
+  - code: "eu-west-1"
+  - code: "eu-west-2"
+  - code: "eu-west-3"
+  - code: "eu-south-1"
+  - code: "eu-south-2"
+  - code: "il-central-1"
+  - code: "me-south-1"
+  - code: "me-central-1"
+  - code: "sa-east-1"

.gitlab/scripts/build_layer.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2021 Datadog, Inc.
+
+# Usage examples :
+# TRACER_VERSION=xxx ARCH=arm64 ./scripts/build_layer.sh
+# ARCH is optional. Default is to build both arm and amd64 layers.
+
+set -e
+
+if [ "$TRACER_VERSION" = "placeholder" ]; then
+    TRACER_VERSION=""
+fi
+
+if [ -z "$TRACER_VERSION" ]; then
+    # Running on dev
+    echo "Running on dev environment"
+    TRACER_VERSION="dev"
+else
+    echo "Found version tag in environment variables"
+    echo "Tracer version: ${TRACER_VERSION}"
+fi
+
+# Move into the root directory, so this script can be called from any directory
+SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+ROOT_DIR=$SCRIPTS_DIR/../..
+cd $ROOT_DIR
+
+LAYER_DIR=".layers"
+TARGET_DIR=$(pwd)/$LAYER_DIR
+DOCKERFILE="./scripts/Dockerfile.r2r"
+
+# Build the image
+function docker_build_zip {
+    arch=$1
+
+    tmp_dir=$(mktemp -d)
+    docker buildx build -t datadog/dd_trace_dotnet:$TRACER_VERSION . \
+        -f $DOCKERFILE \
+        --no-cache \
+        --platform linux/${arch} \
+        --build-arg TRACER_VERSION="${TRACER_VERSION}" \
+        --build-arg ARCH=${arch} \
+        -o $tmp_dir/datadog
+
+    cp $tmp_dir/datadog/dd_trace_dotnet.zip $TARGET_DIR/dd_trace_dotnet_${arch}.zip
+    unzip $tmp_dir/datadog/dd_trace_dotnet.zip -d $TARGET_DIR/dd_trace_dotnet_${arch}
+
+    rm -rf $tmp_dir
+}
+
+# Clean and make directories in ./layers
+function clean_layer_directory {
+    arch=$1
+
+    rm -rf $LAYER_DIR/dd_trace_dotnet_${arch} 2>/dev/null
+    mkdir -p $LAYER_DIR/dd_trace_dotnet_${arch}
+}
+
+echo "Building layers for ${ARCH}"
+clean_layer_directory $ARCH
+docker_build_zip $ARCH
+
+echo "Finished building layers for ${ARCH}"

.gitlab/scripts/download_tracer_artifacts.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+set -e
+
+GITLAB_TOKEN=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name "ci.$CI_PROJECT_NAME.serverless-gitlab-token" \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+TRACER_PROJECT_ID=348
+
+# Clean environment variables if they are set as 'placeholder'
+if [ "$UPSTREAM_PIPELINE_ID" = "placeholder" ]; then
+    UPSTREAM_PIPELINE_ID=""
+fi
+
+if [ "$TRACER_BRANCH" = "placeholder" ]; then
+    TRACER_BRANCH=""
+fi
+
+echo "Running with the following configuration:"
+echo "UPSTREAM_PIPELINE_ID: $UPSTREAM_PIPELINE_ID"
+echo "TRACER_BRANCH: $TRACER_BRANCH"
+
+# If 'UPSTREAM_PIPELINE_ID' or 'TRACER_BRANCH' are not set, exit
+if [ -z "$UPSTREAM_PIPELINE_ID" ] && [ -z "$TRACER_BRANCH" ]; then
+    echo "None of UPSTREAM_PIPELINE_ID or TRACER_BRANCH is set. Exiting..."
+    exit 1
+fi
+
+# If 'UPSTREAM_PIPELINE_ID' is not set, calculate the latest based on the 'TRACER_BRANCH' or,
+# if 'TRACER_BRANCH' is set, prioritize it over 'UPSTREAM_PIPELINE_ID' even if it's set
+#
+# This might happen when doing a manual trigger of the pipeline, normally done for Production deployments with tags.
+if [ -z "$UPSTREAM_PIPELINE_ID" ] || [ -n "$TRACER_BRANCH" ]; then
+    echo "UPSTREAM_PIPELINE_ID is not set, or TRACER_BRANCH is set. Calculating the latest pipeline ID..."
+
+    URL="$CI_API_V4_URL/projects/$TRACER_PROJECT_ID/pipelines?ref=$TRACER_BRANCH&per_page=1&order_by=id&sort=desc"
+    echo "Getting pipelines for '$TRACER_BRANCH' from: $URL"
+    PIPELINES=$(curl $URL --header "PRIVATE-TOKEN: $GITLAB_TOKEN")
+
+    # Get the latest pipeline ID
+    UPSTREAM_PIPELINE_ID=$(echo "${PIPELINES}" | jq -r '.[0] | @base64' | base64 --decode | jq -r '.id')
+fi
+
+echo "UPSTREAM_PIPELINE_ID: $UPSTREAM_PIPELINE_ID"
+
+# Get the jobs of the upstream pipeline
+URL="$CI_API_V4_URL/projects/$TRACER_PROJECT_ID/pipelines/$UPSTREAM_PIPELINE_ID/jobs"
+echo "Looking for the artifacts job 'download-serverless-artifacts' for pipeline ID '$UPSTREAM_PIPELINE_ID' in '$URL'"
+PIPELINE_JOBS=$(curl $URL --header "PRIVATE-TOKEN: $GITLAB_TOKEN")
+
+FOUND_ARTIFACTS_JOB=false
+# Iterate over pipeline trigger jobs
+for pipeline_job in $(echo "${PIPELINE_JOBS}" | jq -r '.[] | @base64'); do
+    # Only check the 'download-serverless-artifacts' job
+    pipeline_job_name=$(echo "${pipeline_job}" | base64 --decode | jq -r '.name')
+    if [ "${pipeline_job_name}" = "download-serverless-artifacts" ]; then
+        FOUND_ARTIFACTS_JOB=true
+        pipeline_job_id=$(echo ${pipeline_job} | base64 --decode | jq -r '.id')
+
+        ARTIFACTS_URL="$CI_API_V4_URL/projects/348/jobs/$pipeline_job_id/artifacts"
+        echo "Downloading artifacts from: $ARTIFACTS_URL"
+        ARTIFACTS=$(curl $ARTIFACTS_URL --header "PRIVATE-TOKEN: $GITLAB_TOKEN" --location --output artifacts.zip)
+    fi
+done
+
+if [ "$FOUND_ARTIFACTS_JOB" = false ]; then
+    echo "No artifacts job found in the pipeline. Exiting..."
+    exit 1
+fi
+
+target_dir=artifacts
+
+mkdir -p $target_dir
+unzip artifacts.zip -d $target_dir
+mv $target_dir/artifacts/* $target_dir
+rmdir $target_dir/artifacts
+
+ls -R $target_dir

.gitlab/scripts/get_secrets.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2024 Datadog, Inc.
+
+set -e
+
+if [ -z "$EXTERNAL_ID_NAME" ]; then
+    printf "[Error] No EXTERNAL_ID_NAME found.\n"
+    printf "Exiting script...\n"
+    exit 1
+fi
+
+if [ -z "$ROLE_TO_ASSUME" ]; then
+    printf "[Error] No ROLE_TO_ASSUME found.\n"
+    printf "Exiting script...\n"
+    exit 1
+fi
+
+printf "Getting AWS External ID...\n"
+
+EXTERNAL_ID=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name "ci.dd-trace-dotnet-aws-lambda-layer.$EXTERNAL_ID_NAME" \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+printf "Getting DD API KEY...\n"
+
+export DD_API_KEY=$(aws ssm get-parameter \
+    --region us-east-1 \
+    --name ci.dd-trace-dotnet-aws-lambda-layer.dd-api-key \
+    --with-decryption \
+    --query "Parameter.Value" \
+    --out text)
+
+printf "Assuming role...\n"
+
+export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \
+    $(aws sts assume-role \
+    --role-arn "arn:aws:iam::$AWS_ACCOUNT:role/$ROLE_TO_ASSUME"  \
+    --role-session-name "ci.dd-trace-dotnet-aws-lambda-layer-$CI_JOB_ID-$CI_JOB_STAGE" \
+    --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
+    --external-id $EXTERNAL_ID \
+    --output text))