From feeb4e431973bc8f117120ab31a2d1e0e52219e6 Mon Sep 17 00:00:00 2001 From: DarcyRaynerDD Date: Thu, 6 May 2021 17:37:02 -0400 Subject: [PATCH] Add CODEOWNERS --- .github/bug_report_template.md | 27 +++++ .github/feature_request_template.md | 27 +++++ CODEOWNERS | 1 + scripts/serverless/Dockerfile | 2 + scripts/serverless/add_new_region.sh | 61 +++++++++++ scripts/serverless/build_binary_and_layer.sh | 42 +++++++ scripts/serverless/build_docker_image.sh | 34 ++++++ scripts/serverless/list_layers.sh | 53 +++++++++ scripts/serverless/publish_layers.sh | 104 ++++++++++++++++++ scripts/serverless/publish_prod.sh | 90 +++++++++++++++ scripts/serverless/publish_sandbox.sh | 17 +++ scripts/serverless/sign_layers.sh | 109 +++++++++++++++++++ 12 files changed, 567 insertions(+) create mode 100644 .github/bug_report_template.md create mode 100644 .github/feature_request_template.md create mode 100644 CODEOWNERS create mode 100644 scripts/serverless/Dockerfile create mode 100755 scripts/serverless/add_new_region.sh create mode 100755 scripts/serverless/build_binary_and_layer.sh create mode 100755 scripts/serverless/build_docker_image.sh create mode 100755 scripts/serverless/list_layers.sh create mode 100755 scripts/serverless/publish_layers.sh create mode 100755 scripts/serverless/publish_prod.sh create mode 100755 scripts/serverless/publish_sandbox.sh create mode 100755 scripts/serverless/sign_layers.sh diff --git a/.github/bug_report_template.md b/.github/bug_report_template.md new file mode 100644 index 00000000..162c07c6 --- /dev/null +++ b/.github/bug_report_template.md @@ -0,0 +1,27 @@ +--- +name: Bug Report +about: Create a report to help us improve +--- + +## Expected Behavior + + +## Actual Behavior + + +## Steps to Reproduce the Problem + + 1. + 1. + 1. + +## Specifications + + - Datadog Lambda Layer version: + - Python version: + +## Stacktrace + + ``` + Paste here + ``` \ No newline at end of file diff --git a/.github/feature_request_template.md b/.github/feature_request_template.md new file mode 100644 index 00000000..ba1fa7ba --- /dev/null +++ b/.github/feature_request_template.md @@ -0,0 +1,27 @@ +--- +name: Feature Request +about: Suggest an idea +--- + +## Expected Behavior + + +## Actual Behavior + + +## Steps to Reproduce the Problem + + 1. + 1. + 1. + +## Specifications + + - Datadog Lambda Layer version: + - Python version: + +## Stacktrace + + ``` + Paste here + ``` \ No newline at end of file diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..e340f1ed --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @DataDog/serverless \ No newline at end of file diff --git a/scripts/serverless/Dockerfile b/scripts/serverless/Dockerfile new file mode 100644 index 00000000..76a04914 --- /dev/null +++ b/scripts/serverless/Dockerfile @@ -0,0 +1,2 @@ +FROM scratch +COPY datadog-agent opt/extensions/datadog-agent \ No newline at end of file diff --git a/scripts/serverless/add_new_region.sh b/scripts/serverless/add_new_region.sh new file mode 100755 index 00000000..f77423d8 --- /dev/null +++ b/scripts/serverless/add_new_region.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2020 Datadog, Inc. + +# Copy layers from us-east-1 to new region +# args: [new-region] + +set -e + +FROM_REGION='us-east-1' + +LAYER_NAMES=("Datadog-Extension") + +NEW_REGION=$1 + +get_max_version() { + layer_name=$1 + region=$2 + last_layer_version=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] | .Version") + if [ "$last_layer_version" == "null" ]; then + echo 0 + else + echo $last_layer_version + fi +} + +if [ -z "$1" ]; then + echo "Region parameter not specified, exiting" + exit 1 +fi + +for layer_name in "${LAYER_NAMES[@]}"; do + # get latest version + last_layer_version=$(get_max_version $layer_name $FROM_REGION) + starting_version=$(get_max_version $layer_name $NEW_REGION) + starting_version=$(expr $starting_version + 1) + + # exit if region is already all caught up + if [ $starting_version -ge $last_layer_version ]; then + echo "INFO: $NEW_REGION is already up to date for $layer_name" + continue + fi + + # run for each version of layer + for i in $(seq 1 $last_layer_version); do + layer_path=$layer_name"_"$i.zip + + # download layer versions + URL=$(AWS_REGION=$FROM_REGION aws lambda get-layer-version --layer-name $layer_name --version-number $i --query Content.Location --output text) + curl $URL -o $layer_path + + # publish layer to new region + ./publish_layer + + publish_layer $NEW_REGION + rm $layer_path + done +done diff --git a/scripts/serverless/build_binary_and_layer.sh b/scripts/serverless/build_binary_and_layer.sh new file mode 100755 index 00000000..35794ab2 --- /dev/null +++ b/scripts/serverless/build_binary_and_layer.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2020 Datadog, Inc. + +# Builds Datadogpy layers for lambda functions, using Docker +set -e + +# Move into the root directory, so this script can be called from any directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/.. + +LAYER_DIR=".layers" +LAYER_FILE="datadog_extension" +EXTENSION_DIR="extensions" +TMP_DIR='./var/task' + +rm -rf $LAYER_DIR +rm -rf $EXTENSION_DIR +rm -rf $TMP_DIR +mkdir $LAYER_DIR +mkdir $EXTENSION_DIR +mkdir -p $TMP_DIR + +TARGET_DIR=$(pwd)/$EXTENSION_DIR + +echo "Building Lambda extension binary" +cd ~/dd/datadog-agent/cmd/serverless +GOOS=linux go build -ldflags="-s -w" -tags serverless -o $TARGET_DIR/datadog-agent +if [ "$COMPRESS" = true ]; then + upx --brute $TARGET_DIR/datadog-agent +fi + +cd - +rm -rf "./var" + +echo "Building Lambda layer" +zip -q -r "${LAYER_DIR}/${LAYER_FILE}" -r $EXTENSION_DIR + +echo "Done!" diff --git a/scripts/serverless/build_docker_image.sh b/scripts/serverless/build_docker_image.sh new file mode 100755 index 00000000..45736ce5 --- /dev/null +++ b/scripts/serverless/build_docker_image.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Use with `VERSION= ./build_docker_image.sh` + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2020 Datadog, Inc. + +set -e + +DOCKER_REPOSITORY_NAME="datadog/lambda-extension" +DOCKERFILE_LOCATION="build-scripts/serverless/Dockerfile" + +# Move into the root directory, so this script can be called from any directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/.. + +if [ -z "$VERSION" ]; then + echo "Version not specified" + echo "" + echo "EXITING SCRIPT." + exit 1 +fi + +# Build the image, tagged with the version +echo "Building the Docker image" +docker build extensions \ + -f $DOCKERFILE_LOCATION \ + -t $DOCKER_REPOSITORY_NAME:$VERSION \ + --no-cache + +# Also tag the image with :latest +docker tag $DOCKER_REPOSITORY_NAME:$VERSION $DOCKER_REPOSITORY_NAME:latest \ No newline at end of file diff --git a/scripts/serverless/list_layers.sh b/scripts/serverless/list_layers.sh new file mode 100755 index 00000000..5d401ad2 --- /dev/null +++ b/scripts/serverless/list_layers.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2020 Datadog, Inc. + +# Lists most recent layers ARNs across regions to STDOUT +# Optionals args: [layer-name] [region] + +set -e + +LAYERS=("Datadog-Extension") +AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') +LAYERS_MISSING_REGIONS=() + +# Check region arg +if [ -z "$2" ]; then + >&2 echo "Region parameter not specified, running for all available regions." + REGIONS=$AVAILABLE_REGIONS +else + + >&2 echo "Region parameter specified: $2" + if [[ ! "$AVAILABLE_REGIONS" == *"$2"* ]]; then + >&2 echo "Could not find $2 in available regions:" $AVAILABLE_REGIONS + >&2 echo "" + >&2 echo "EXITING SCRIPT." + exit 1 + fi + REGIONS=($2) +fi + +for region in $REGIONS +do + for layer_name in "${LAYERS[@]}" + do + last_layer_arn=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] | .LayerVersionArn") + if [ "$last_layer_arn" == "null" ]; then + >&2 echo "No layer found for $region, $layer_name" + if [[ ! " ${LAYERS_MISSING_REGIONS[@]} " =~ " ${region} " ]]; then + LAYERS_MISSING_REGIONS+=( $region ) + fi + else + echo $last_layer_arn + fi + done +done + +if [ ${#LAYERS_MISSING_REGIONS[@]} -gt 0 ]; then + echo "WARNING: Following regions missing layers: ${LAYERS_MISSING_REGIONS[@]}" + echo "Please run ./add_new_region.sh to add layers to the missing regions" + exit 1 +fi diff --git a/scripts/serverless/publish_layers.sh b/scripts/serverless/publish_layers.sh new file mode 100755 index 00000000..43ee2b89 --- /dev/null +++ b/scripts/serverless/publish_layers.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2020 Datadog, Inc. + +# Publish the datadog lambda layer across regions, using the AWS CLI +# Usage: VERSION=5 REGIONS=us-east-1 publish_layers.sh +# VERSION is required. +set -e + +# Move into the root directory, so this script can be called from any directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/.. + +LAYER_PATH=".layers/datadog_extension.zip" +LAYER_NAME="Datadog-Extension" +AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName') + +# Check that the layer files exist +if [ ! -f $LAYER_PATH ]; then + echo "Could not find $LAYER_PATH." + exit 1 +fi + +# Determine the target regions +if [ -z "$REGIONS" ]; then + echo "Region not specified, running for all available regions." + REGIONS=$AVAILABLE_REGIONS +else + echo "Region specified: $REGIONS" + if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then + echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS" + echo "" + echo "EXITING SCRIPT." + exit 1 + fi +fi + +# Determine the target layer version +if [ -z "$VERSION" ]; then + echo "Layer version not specified" + echo "" + echo "EXITING SCRIPT." + exit 1 +else + echo "Layer version specified: $VERSION" +fi + +read -p "Ready to publish layer $LAYER_NAME version $VERSION to regions ${REGIONS[*]} (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi + +publish_layer() { + region=$1 + version_nbr=$(aws lambda publish-layer-version --layer-name $LAYER_NAME \ + --description "Datadog Lambda Extension" \ + --zip-file "fileb://$LAYER_PATH" \ + --region $region | jq -r '.Version') + + permission=$(aws lambda add-layer-version-permission --layer-name $LAYER_NAME \ + --version-number $version_nbr \ + --statement-id "release-$version_nbr" \ + --action lambda:GetLayerVersion --principal "*" \ + --region $region) + + echo $version_nbr +} + +for region in $REGIONS +do + echo "Starting publishing layer for region $region..." + + latest_version=$(aws lambda list-layer-versions --region $region --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`') + if [ $latest_version -ge $VERSION ]; then + echo "Layer $LAYER_NAME version $VERSION already exists in region $region, skipping..." + continue + elif [ $latest_version -lt $((VERSION-1)) ]; then + read -p "WARNING: The latest version of layer $LAYER_NAME in region $region is $latest_version, publish all the missing versions including $VERSION or EXIT the script (y/n)?" CONT + if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 + fi + fi + + while [ $latest_version -lt $VERSION ]; do + latest_version=$(publish_layer $region) + echo "Published version $latest_version for layer $LAYER_NAME in region $region" + + # This shouldn't happen unless someone manually deleted the latest version, say 28, and + # then tries to republish 28 again. The published version would actually be 29, because + # Lambda layers are immutable and AWS will skip deleted version and use the next number. + if [ $latest_version -gt $VERSION ]; then + echo "ERROR: Published version $latest_version is greater than the desired version $VERSION!" + echo "Exiting" + exit 1 + fi + done +done + +echo "Done !" diff --git a/scripts/serverless/publish_prod.sh b/scripts/serverless/publish_prod.sh new file mode 100755 index 00000000..a5bc91c7 --- /dev/null +++ b/scripts/serverless/publish_prod.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2020 Datadog, Inc. + +# Use with `VERSION= ./build_docker_image.sh` + +set -e + +DOCKER_REPOSITORY_NAME="datadog/lambda-extension" + +# Ensure the target extension version is defined +if [ -z "$VERSION" ]; then + echo "Extension version not specified" + echo "" + echo "EXITING SCRIPT." + exit 1 +fi + +# Ensure we are releasing the correct commit +cd ~/dd/datadog-agent +if [[ `git status --porcelain` ]]; then + echo "Detected uncommitted changes on datadog-agent branch, aborting" + exit 1 +fi + +CURRENT_SHA=$(git rev-parse HEAD) +COMMIT_MESSAGE=$(git log -1 --pretty=%B) +echo "Current commit: $CURRENT_SHA" +echo "Current commit message: $COMMIT_MESSAGE" +echo +read -p "Ready to publish commit $CURRENT_SHA as Datadog Lambda Extension version $VERSION (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi +cd - + +# Move into the script directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR + +# Manually confirm access +read -p "Please confirm that you have write access to the datadog-agent repository in GitHub. (y/n)" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi +read -p "Please confirm that you have push access to the datadog/lambda-extension repository in Dockerhub. (y/n)" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi + +docker login + +echo "Checking that you have access to the commercial AWS account" +aws-vault exec prod-engineering -- aws sts get-caller-identity + +echo "Checking that you have access to the GovCloud AWS account" +saml2aws login -a govcloud-us1-fed-human-engineering +AWS_PROFILE=govcloud-us1-fed-human-engineering aws sts get-caller-identity + +COMPRESS=true ./build_binary_and_layer.sh +./build_docker_image.sh + +echo "Signing the layer" +aws-vault exec prod-engineering -- ./sign_layers.sh prod + +echo "Publishing layers to commercial AWS regions" +aws-vault exec prod-engineering --no-session -- ./publish_layers.sh + +echo "Publishing layers to GovCloud AWS regions" +saml2aws login -a govcloud-us1-fed-human-engineering +AWS_PROFILE=govcloud-us1-fed-human-engineering ./publish_layers.sh + +echo "Pushing Docker image to Dockerhub" +docker push $DOCKER_REPOSITORY_NAME:$VERSION +docker push $DOCKER_REPOSITORY_NAME:latest + +echo "Creating tag for release on GitHub" +git tag "lambda-extension-$VERSION" +git push origin "refs/tags/lambda-extension-$VERSION" + +echo "New extension version published to AWS and Dockerhub!" +echo +echo "Please create a new GitHub release with the tag lambda-extension-${VERSION}" +echo "https://github.com/DataDog/datadog-agent/releases/new?tag=lambda-extension-$VERSION&title=lambda-extension-$VERSION" \ No newline at end of file diff --git a/scripts/serverless/publish_sandbox.sh b/scripts/serverless/publish_sandbox.sh new file mode 100755 index 00000000..f702e9bd --- /dev/null +++ b/scripts/serverless/publish_sandbox.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2020 Datadog, Inc. + +#!/bin/bash +set -e + +# Move into the script directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR + +./build_binary_and_layer.sh +aws-vault exec sandbox-account-admin -- ./sign_layers.sh sandbox +REGIONS=sa-east-1 aws-vault exec sandbox-account-admin -- ./publish_layers.sh diff --git a/scripts/serverless/sign_layers.sh b/scripts/serverless/sign_layers.sh new file mode 100755 index 00000000..b4cf1a00 --- /dev/null +++ b/scripts/serverless/sign_layers.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019 Datadog, Inc. + +set -e + +# Move into the root directory, so this script can be called from any directory +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd $DIR/.. + +LAYER_DIR=".layers" +LAYER_FILES=("datadog_extension.zip") +SIGNING_PROFILE_NAME="DatadogLambdaSigningProfile" + +# Check account parameter +VALID_ACCOUNTS=("sandbox" "prod") +if [ -z "$1" ]; then + echo "ERROR: You must pass an account parameter to sign the layers" + exit 1 +fi +if [[ ! "${VALID_ACCOUNTS[@]}" =~ $1 ]]; then + echo "ERROR: The account parameter was invalid. Please choose sandbox or prod." + exit 1 +fi +if [ "$1" = "sandbox" ]; then + REGION="sa-east-1" + S3_BUCKET_NAME="dd-lambda-signing-bucket-sandbox" +fi +if [ "$1" = "prod" ]; then + REGION="us-east-1" + S3_BUCKET_NAME="dd-lambda-signing-bucket" +fi + +for LAYER_FILE in "${LAYER_FILES[@]}" +do + echo + echo "${LAYER_FILE}" + echo "-------------------------" + + LAYER_LOCAL_PATH="${LAYER_DIR}/${LAYER_FILE}" + + # Upload the layer to S3 for signing + echo "Uploading layer to S3 for signing..." + UUID=$(uuidgen) + S3_UNSIGNED_ZIP_KEY="${UUID}.zip" + S3_UNSIGNED_ZIP_URI="s3://${S3_BUCKET_NAME}/${S3_UNSIGNED_ZIP_KEY}" + aws s3 cp $LAYER_LOCAL_PATH $S3_UNSIGNED_ZIP_URI + + # Start a signing job + echo "Starting the signing job..." + SIGNING_JOB_ID=$(aws signer start-signing-job \ + --source "s3={bucketName=${S3_BUCKET_NAME},key=${S3_UNSIGNED_ZIP_KEY},version=null}" \ + --destination "s3={bucketName=${S3_BUCKET_NAME}}" \ + --profile-name $SIGNING_PROFILE_NAME \ + --region $REGION \ + | jq -r '.jobId'\ + ) + + # Wait for the signing job to complete + echo "Waiting for the signing job to complete..." + SECONDS_WAITED_SO_FAR=0 + while : + do + sleep 3 + SECONDS_WAITED_SO_FAR=$((SECONDS_WAITED_SO_FAR + 3)) + + SIGNING_JOB_DESCRIPTION=$(aws signer describe-signing-job \ + --job-id $SIGNING_JOB_ID \ + --region $REGION\ + ) + SIGNING_JOB_STATUS=$(echo $SIGNING_JOB_DESCRIPTION | jq -r '.status') + SIGNING_JOB_STATUS_REASON=$(echo $SIGNING_JOB_DESCRIPTION | jq -r '.statusReason') + + if [ $SIGNING_JOB_STATUS = "Succeeded" ]; then + echo "Signing job succeeded!" + break + fi + + if [ $SIGNING_JOB_STATUS = "Failed" ]; then + echo "ERROR: Signing job failed" + echo $SIGNING_JOB_STATUS_REASON + exit 1 + fi + + if [ $SECONDS_WAITED_SO_FAR -ge 60 ]; then + echo "ERROR: Timed out waiting for the signing job to complete" + exit 1 + fi + + echo "Signing job still in progress..." + done + + # Download the signed ZIP, overwriting the original ZIP + echo "Replacing the local layer with the signed layer from S3..." + S3_SIGNED_ZIP_KEY="${SIGNING_JOB_ID}.zip" + S3_SIGNED_ZIP_URI="s3://${S3_BUCKET_NAME}/${S3_SIGNED_ZIP_KEY}" + aws s3 cp $S3_SIGNED_ZIP_URI $LAYER_LOCAL_PATH + + # Delete the signed and unsigned ZIPs in S3 + echo "Cleaning up the S3 bucket..." + aws s3api delete-object --bucket $S3_BUCKET_NAME --key $S3_UNSIGNED_ZIP_KEY + aws s3api delete-object --bucket $S3_BUCKET_NAME --key $S3_SIGNED_ZIP_KEY +done + +echo +echo "Successfully signed all layers!"