Add CODEOWNERS

.github/bug_report_template.md

@@ -0,0 +1,27 @@
+---
+name: Bug Report
+about: Create a report to help us improve
+---
+
+## Expected Behavior
+
+
+## Actual Behavior
+
+
+## Steps to Reproduce the Problem
+
+  1.
+  1.
+  1.
+
+## Specifications
+
+  - Datadog Lambda Layer version: 
+  - Python version: 
+
+## Stacktrace
+
+  ```
+  Paste here
+  ```

.github/feature_request_template.md

@@ -0,0 +1,27 @@
+---
+name: Feature Request
+about: Suggest an idea
+---
+
+## Expected Behavior
+
+
+## Actual Behavior
+
+
+## Steps to Reproduce the Problem
+
+  1.
+  1.
+  1.
+
+## Specifications
+
+  - Datadog Lambda Layer version: 
+  - Python version: 
+
+## Stacktrace
+
+  ```
+  Paste here
+  ```

CODEOWNERS

@@ -0,0 +1 @@
+* @DataDog/serverless

scripts/serverless/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+COPY datadog-agent opt/extensions/datadog-agent

scripts/serverless/add_new_region.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2020 Datadog, Inc.
+
+# Copy layers from us-east-1 to new region
+# args: [new-region]
+
+set -e
+
+FROM_REGION='us-east-1'
+
+LAYER_NAMES=("Datadog-Extension")
+
+NEW_REGION=$1
+
+get_max_version() {
+    layer_name=$1
+    region=$2
+    last_layer_version=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] |  .Version")
+    if [ "$last_layer_version" == "null" ]; then
+        echo 0
+    else
+        echo $last_layer_version
+    fi
+}
+
+if [ -z "$1" ]; then
+    echo "Region parameter not specified, exiting"
+    exit 1
+fi
+
+for layer_name in "${LAYER_NAMES[@]}"; do
+    # get latest version
+    last_layer_version=$(get_max_version $layer_name $FROM_REGION)
+    starting_version=$(get_max_version $layer_name $NEW_REGION)
+    starting_version=$(expr $starting_version + 1)
+
+    # exit if region is already all caught up
+    if [ $starting_version -ge $last_layer_version ]; then
+        echo "INFO: $NEW_REGION is already up to date for $layer_name"
+        continue
+    fi
+
+    # run for each version of layer
+    for i in $(seq 1 $last_layer_version); do
+        layer_path=$layer_name"_"$i.zip
+
+        # download layer versions
+        URL=$(AWS_REGION=$FROM_REGION aws lambda get-layer-version --layer-name $layer_name --version-number $i --query Content.Location --output text)
+        curl $URL -o $layer_path
+
+        # publish layer to new region
+        ./publish_layer
+
+        publish_layer $NEW_REGION
+        rm $layer_path
+    done
+done

scripts/serverless/build_binary_and_layer.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2020 Datadog, Inc.
+
+# Builds Datadogpy layers for lambda functions, using Docker
+set -e
+
+# Move into the root directory, so this script can be called from any directory
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd $DIR/..
+
+LAYER_DIR=".layers"
+LAYER_FILE="datadog_extension"
+EXTENSION_DIR="extensions"
+TMP_DIR='./var/task'
+
+rm -rf $LAYER_DIR
+rm -rf $EXTENSION_DIR
+rm -rf $TMP_DIR
+mkdir $LAYER_DIR
+mkdir $EXTENSION_DIR
+mkdir -p $TMP_DIR
+
+TARGET_DIR=$(pwd)/$EXTENSION_DIR
+
+echo "Building Lambda extension binary"
+cd ~/dd/datadog-agent/cmd/serverless
+GOOS=linux go build -ldflags="-s -w" -tags serverless -o $TARGET_DIR/datadog-agent
+if [ "$COMPRESS" = true ]; then
+    upx --brute $TARGET_DIR/datadog-agent
+fi
+
+cd -
+rm -rf "./var"
+
+echo "Building Lambda layer"
+zip -q -r "${LAYER_DIR}/${LAYER_FILE}" -r $EXTENSION_DIR
+
+echo "Done!"

scripts/serverless/build_docker_image.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Use with `VERSION=<DESIRED_VERSION> ./build_docker_image.sh`
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2020 Datadog, Inc.
+
+set -e
+
+DOCKER_REPOSITORY_NAME="datadog/lambda-extension"
+DOCKERFILE_LOCATION="build-scripts/serverless/Dockerfile"
+
+# Move into the root directory, so this script can be called from any directory
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd $DIR/..
+
+if [ -z "$VERSION" ]; then
+    echo "Version not specified"
+    echo ""
+    echo "EXITING SCRIPT."
+    exit 1
+fi
+
+# Build the image, tagged with the version
+echo "Building the Docker image"
+docker build extensions \
+  -f $DOCKERFILE_LOCATION \
+  -t $DOCKER_REPOSITORY_NAME:$VERSION \
+  --no-cache
+
+# Also tag the image with :latest
+docker tag $DOCKER_REPOSITORY_NAME:$VERSION $DOCKER_REPOSITORY_NAME:latest

scripts/serverless/list_layers.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2020 Datadog, Inc.
+
+# Lists most recent layers ARNs across regions to STDOUT
+# Optionals args: [layer-name] [region]
+
+set -e
+
+LAYERS=("Datadog-Extension")
+AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
+LAYERS_MISSING_REGIONS=()
+
+# Check region arg
+if [ -z "$2" ]; then
+    >&2 echo "Region parameter not specified, running for all available regions."
+    REGIONS=$AVAILABLE_REGIONS
+else
+
+    >&2 echo "Region parameter specified: $2"
+    if [[ ! "$AVAILABLE_REGIONS" == *"$2"* ]]; then
+        >&2 echo "Could not find $2 in available regions:" $AVAILABLE_REGIONS
+        >&2 echo ""
+        >&2 echo "EXITING SCRIPT."
+        exit 1
+    fi
+    REGIONS=($2)
+fi
+
+for region in $REGIONS
+do
+    for layer_name in "${LAYERS[@]}"
+    do
+        last_layer_arn=$(aws lambda list-layer-versions --layer-name $layer_name --region $region | jq -r ".LayerVersions | .[0] |  .LayerVersionArn")
+        if [ "$last_layer_arn" == "null" ]; then
+            >&2 echo "No layer found for $region, $layer_name"
+            if [[ ! " ${LAYERS_MISSING_REGIONS[@]} " =~ " ${region} " ]]; then
+                LAYERS_MISSING_REGIONS+=( $region )
+            fi
+        else
+            echo $last_layer_arn
+        fi
+    done
+done
+
+if [ ${#LAYERS_MISSING_REGIONS[@]} -gt 0 ]; then
+    echo "WARNING: Following regions missing layers: ${LAYERS_MISSING_REGIONS[@]}"
+    echo "Please run ./add_new_region.sh <new_region> to add layers to the missing regions"
+    exit 1
+fi

scripts/serverless/publish_layers.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2020 Datadog, Inc.
+
+# Publish the datadog lambda layer across regions, using the AWS CLI
+# Usage: VERSION=5 REGIONS=us-east-1 publish_layers.sh
+# VERSION is required.
+set -e
+
+# Move into the root directory, so this script can be called from any directory
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd $DIR/..
+
+LAYER_PATH=".layers/datadog_extension.zip"
+LAYER_NAME="Datadog-Extension"
+AVAILABLE_REGIONS=$(aws ec2 describe-regions | jq -r '.[] | .[] | .RegionName')
+
+# Check that the layer files exist
+if [ ! -f $LAYER_PATH  ]; then
+    echo "Could not find $LAYER_PATH."
+    exit 1
+fi
+
+# Determine the target regions
+if [ -z "$REGIONS" ]; then
+    echo "Region not specified, running for all available regions."
+    REGIONS=$AVAILABLE_REGIONS
+else
+    echo "Region specified: $REGIONS"
+    if [[ ! "$AVAILABLE_REGIONS" == *"$REGIONS"* ]]; then
+        echo "Could not find $REGIONS in available regions: $AVAILABLE_REGIONS"
+        echo ""
+        echo "EXITING SCRIPT."
+        exit 1
+    fi
+fi
+
+# Determine the target layer version
+if [ -z "$VERSION" ]; then
+    echo "Layer version not specified"
+    echo ""
+    echo "EXITING SCRIPT."
+    exit 1
+else
+    echo "Layer version specified: $VERSION"
+fi
+
+read -p "Ready to publish layer $LAYER_NAME version $VERSION to regions ${REGIONS[*]} (y/n)?" CONT
+if [ "$CONT" != "y" ]; then
+    echo "Exiting"
+    exit 1
+fi
+
+publish_layer() {
+    region=$1
+    version_nbr=$(aws lambda publish-layer-version --layer-name $LAYER_NAME \
+        --description "Datadog Lambda Extension" \
+        --zip-file "fileb://$LAYER_PATH" \
+        --region $region | jq -r '.Version')
+
+    permission=$(aws lambda add-layer-version-permission --layer-name $LAYER_NAME \
+        --version-number $version_nbr \
+        --statement-id "release-$version_nbr" \
+        --action lambda:GetLayerVersion --principal "*" \
+        --region $region)
+
+    echo $version_nbr
+}
+
+for region in $REGIONS
+do
+    echo "Starting publishing layer for region $region..."
+
+    latest_version=$(aws lambda list-layer-versions --region $region --layer-name $LAYER_NAME --query 'LayerVersions[0].Version || `0`')
+    if [ $latest_version -ge $VERSION ]; then
+        echo "Layer $LAYER_NAME version $VERSION already exists in region $region, skipping..."
+        continue
+    elif [ $latest_version -lt $((VERSION-1)) ]; then
+        read -p "WARNING: The latest version of layer $LAYER_NAME in region $region is $latest_version, publish all the missing versions including $VERSION or EXIT the script (y/n)?" CONT
+        if [ "$CONT" != "y" ]; then
+            echo "Exiting"
+            exit 1
+        fi
+    fi
+
+    while [ $latest_version -lt $VERSION ]; do
+        latest_version=$(publish_layer $region)
+        echo "Published version $latest_version for layer $LAYER_NAME in region $region"
+
+        # This shouldn't happen unless someone manually deleted the latest version, say 28, and
+        # then tries to republish 28 again. The published version would actually be 29, because
+        # Lambda layers are immutable and AWS will skip deleted version and use the next number. 
+        if [ $latest_version -gt $VERSION ]; then
+            echo "ERROR: Published version $latest_version is greater than the desired version $VERSION!"
+            echo "Exiting"
+            exit 1
+        fi
+    done
+done
+
+echo "Done !"