Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GODRIVER-2792 Migrate CI to RHEL 8.7 #1362

Merged
merged 13 commits into from
Sep 12, 2023
Merged

GODRIVER-2792 Migrate CI to RHEL 8.7 #1362

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3c4e203
GODRIVER-2792 Migrate CI to RHEL 8.0
prestonvasquez Aug 25, 2023
98e963c
GODRIVER-2792 Migrate AWS to RHEL 8.0
prestonvasquez Aug 25, 2023
c8efe4c
GODRIVER-2792 Migrate KMS to RHEL 8.0
prestonvasquez Aug 25, 2023
463834a
GODRIVER-2792 Add comments for az debian migration
prestonvasquez Aug 25, 2023
e6c0d12
GODRIVER-2792 Bump RHEL to 8.7
prestonvasquez Aug 31, 2023
f51b88d
Merge branch 'master' into GODRIVER-2792
prestonvasquez Sep 11, 2023
18d61af
GODRIVER-2792 Update KMS to 8.7
prestonvasquez Sep 11, 2023
6cfe71f
GODRIVER-2792 Move ECS-dependencies down to U 18.04
prestonvasquez Sep 11, 2023
d859b77
GODRIVER-2792 Switch KMS to Debian
prestonvasquez Sep 11, 2023
7fd65b7
GODRIVER-2792 Include RHEL 8.7 for 3.6 server
prestonvasquez Sep 11, 2023
e35578d
GODRIVER-2792 Fix libmongocrypt for kms
prestonvasquez Sep 11, 2023
983da30
GODRIVER-2792 Update EXPECT_ERROR for AWS KMS test
prestonvasquez Sep 12, 2023
17893eb
GODRIVER-2792 Update BUILD comment
prestonvasquez Sep 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 47 additions & 46 deletions .evergreen/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,8 @@ functions:
export UPLOAD_BUCKET="$UPLOAD_BUCKET"
export PROJECT="$PROJECT"
export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
export PATH="$PATH"
EOT
# See what we variables we've set.
Expand Down Expand Up @@ -245,6 +245,7 @@ functions:
params:
shell: "bash"
script: |
set -x
${PREPARE_SHELL}

MONGODB_VERSION=${VERSION} \
Expand Down Expand Up @@ -2082,7 +2083,7 @@ tasks:
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
tar czf testgcpkms.tgz ./testkms ./install/libmongocrypt/lib/libmongocrypt.*
tar czf testgcpkms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.*
GCPKMS_SRC=testgcpkms.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
echo "Copying files ... end"

Expand All @@ -2101,7 +2102,7 @@ tasks:
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
export GCPKMS_ZONE=${GCPKMS_ZONE}
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
GCPKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib MONGODB_URI='mongodb://localhost:27017' PROVIDER='gcp' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
GCPKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='gcp' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh

- name: "testgcpkms-fail-task"
# testgcpkms-fail-task runs in a non-GCE environment.
Expand All @@ -2119,7 +2120,7 @@ tasks:
PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
make build-kms-test
echo "Building build-kms-test ... end"
LD_LIBRARY_PATH=./install/libmongocrypt/lib \
LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
blink1073 marked this conversation as resolved.
Show resolved Hide resolved
MONGODB_URI='mongodb://localhost:27017/' \
EXPECT_ERROR='unable to retrieve GCP credentials' \
PROVIDER='gcp' \
Expand All @@ -2143,7 +2144,7 @@ tasks:
export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"

LD_LIBRARY_PATH=./install/libmongocrypt/lib \
LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
MONGODB_URI='${atlas_free_tier_uri}' \
PROVIDER='aws' \
./testkms
Expand All @@ -2165,9 +2166,9 @@ tasks:
make build-kms-test
echo "Building build-kms-test ... end"

LD_LIBRARY_PATH=./install/libmongocrypt/lib \
LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
MONGODB_URI='${atlas_free_tier_uri}' \
EXPECT_ERROR='unable to retrieve aws credentials' \
EXPECT_ERROR='status=400' \
PROVIDER='aws' \
./testkms

Expand All @@ -2191,7 +2192,7 @@ tasks:
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
echo '${testazurekms_privatekey}' > /tmp/testazurekms.prikey
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms.prikey
tar czf testazurekms.tgz ./testkms ./install/libmongocrypt/lib/libmongocrypt.*
tar czf testazurekms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.*
AZUREKMS_SRC=testazurekms.tgz AZUREKMS_DST=/tmp $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
echo "Copying files ... end"
echo "Untarring file ... begin"
Expand All @@ -2209,7 +2210,7 @@ tasks:
export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
echo '${testazurekms_privatekey}' > /tmp/testazurekms.prikey
export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms.prikey
AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh

- name: "testazurekms-fail-task"
# testazurekms-fail-task runs without environment variables.
Expand All @@ -2228,7 +2229,7 @@ tasks:
make build-kms-test
echo "Building build-kms-test ... end"

LD_LIBRARY_PATH=./install/libmongocrypt/lib \
LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
MONGODB_URI='mongodb://localhost:27017' \
EXPECT_ERROR='unable to retrieve azure credentials' \
PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' \
Expand Down Expand Up @@ -2311,9 +2312,9 @@ axes:
GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
GO_DIST: "C:\\golang\\go1.20"
VENV_BIN_DIR: "Scripts"
- id: "ubuntu1604-64-go-1-20"
display_name: "Ubuntu 16.04"
run_on: ubuntu1604-build
- id: "rhel87-64-go-1-20"
display_name: "RHEL 8.7"
run_on: rhel8.7-large
variables:
GO_DIST: "/opt/golang/go1.20"
- id: "macos11-go-1-20"
Expand All @@ -2335,9 +2336,9 @@ axes:
GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
GO_DIST: "C:\\golang\\go1.20"
VENV_BIN_DIR: "Scripts"
- id: "ubuntu1804-64-go-1-20"
display_name: "Ubuntu 18.04"
run_on: ubuntu1804-build
- id: "rhel87-64-go-1-20"
display_name: "RHEL 8.7"
run_on: rhel8.7-large
variables:
GO_DIST: "/opt/golang/go1.20"
- id: "macos11-go-1-20"
Expand All @@ -2347,13 +2348,12 @@ axes:
variables:
GO_DIST: "/opt/golang/go1.20"

# OCSP linux tasks need to run against this OS since stapling is disabled on Ubuntu 18.04 (SERVER-51364)
- id: ocsp-rhel-70
- id: ocsp-rhel-87
display_name: OS
values:
- id: "rhel70-go-1-20"
display_name: "RHEL 7.0"
run_on: rhel70-build
- id: "rhel87-go-1-20"
display_name: "RHEL 8.7"
run_on: rhel8.7-large
variables:
GO_DIST: "/opt/golang/go1.20"

Expand All @@ -2368,6 +2368,8 @@ axes:
GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
GO_DIST: "C:\\golang\\go1.20"
SKIP_ECS_AUTH_TEST: true
# TODO(BUILD-17329): Update this to RHEL 8.7 after we add a new ECS task
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will actually be Ubuntu 22.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ticket says "We would like to add a new task definition that targets RHEL 8." I thought the goal was to move away from Ubuntu?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The build team pushed back because it was easier to target Ubuntu on ECS.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're still using Ubuntu for some things in the Python toolchain, the main point of moving to RHEL was for wide support for server versions.

# definition that targets RHEL 8.
- id: "ubuntu1804-64-go-1-20"
display_name: "Ubuntu 18.04"
run_on: ubuntu1804-test
Expand All @@ -2386,18 +2388,18 @@ axes:
- id: os-faas-80
display_name: OS
values:
- id: "rhel80-large-go-1-20"
display_name: "RHEL 8.0"
run_on: rhel80-large
- id: "rhel87-large-go-1-20"
display_name: "RHEL 8.7"
run_on: rhel8.7-large
variables:
GO_DIST: "/opt/golang/go1.20"

- id: os-serverless
display_name: OS
values:
- id: "ubuntu2204-go-1-20"
display_name: "Ubuntu 22.04"
run_on: ubuntu2204-small
- id: "rhel87-go-1-20"
display_name: "RHEL 8.7"
run_on: rhel8.7-small
variables:
GO_DIST: "/opt/golang/go1.20"

Expand Down Expand Up @@ -2620,7 +2622,7 @@ buildvariants:
- name: static-analysis
display_name: "Static Analysis"
run_on:
- ubuntu1804-build
- rhel8.7-large
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
Expand All @@ -2629,7 +2631,7 @@ buildvariants:
- name: perf
display_name: "Performance"
run_on:
- ubuntu1804-build
- rhel8.7-large
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
Expand All @@ -2638,16 +2640,16 @@ buildvariants:
- name: build-check
display_name: "Compile Only Checks"
run_on:
- ubuntu1804-test
- rhel8.7-large
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
- name: ".compile-check"

- name: atlas-test
display_name: "Atlas test"
run_on:
- ubuntu1804-build
- rhel8.7-large
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
Expand All @@ -2656,7 +2658,7 @@ buildvariants:
- name: atlas-data-lake-test
display_name: "Atlas Data Lake Test"
run_on:
- ubuntu1804-build
- rhel8.7-large
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
Expand Down Expand Up @@ -2693,8 +2695,8 @@ buildvariants:
- name: "aws-auth-test"

- matrix_name: "ocsp-test"
matrix_spec: { version: ["4.4", "5.0", "6.0", "7.0", "latest"], ocsp-rhel-70: ["rhel70-go-1-20"] }
display_name: "OCSP ${version} ${ocsp-rhel-70}"
matrix_spec: { version: ["4.4", "5.0", "6.0", "7.0", "latest"], ocsp-rhel-87: ["rhel87-go-1-20"] }
display_name: "OCSP ${version} ${ocsp-rhel-87}"
batchtime: 20160 # Use a batchtime of 14 days as suggested by the OCSP test README
tasks:
- name: ".ocsp"
Expand All @@ -2716,7 +2718,7 @@ buildvariants:
- name: ".ocsp-rsa !.ocsp-staple"

- matrix_name: "race-test"
matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
matrix_spec: { version: ["latest"], os-ssl-40: ["rhel87-64-go-1-20"] }
display_name: "Race Detector Test"
tasks:
- name: ".race"
Expand All @@ -2728,14 +2730,13 @@ buildvariants:
- name: ".versioned-api"

- matrix_name: "kms-tls-test"
matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
matrix_spec: { version: ["latest"], os-ssl-40: ["rhel87-64-go-1-20"] }
display_name: "KMS TLS ${os-ssl-40}"
tasks:
- name: ".kms-tls"

- matrix_name: "load-balancer-test"
# The LB software is only available on Ubuntu 18.04, so we don't test on all OSes.
matrix_spec: { version: ["5.0", "6.0", "7.0", "latest", "rapid"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
matrix_spec: { version: ["5.0", "6.0", "7.0", "latest", "rapid"], os-ssl-40: ["rhel87-64-go-1-20"] }
display_name: "Load Balancer Support ${version} ${os-ssl-40}"
tasks:
- name: ".load-balancer"
Expand All @@ -2747,28 +2748,28 @@ buildvariants:
- "serverless_task_group"

- matrix_name: "kms-kmip-test"
matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
matrix_spec: { version: ["latest"], os-ssl-40: ["rhel87-64-go-1-20"] }
display_name: "KMS KMIP ${os-ssl-40}"
tasks:
- name: ".kms-kmip"

- matrix_name: "fuzz-test"
matrix_spec: { version: ["5.0"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
matrix_spec: { version: ["5.0"], os-ssl-40: ["rhel87-64-go-1-20"] }
display_name: "Fuzz ${version} ${os-ssl-40}"
tasks:
- name: "test-fuzz"
batchtime: 1440 # Run at most once per 24 hours.

- matrix_name: "faas-test"
matrix_spec: { version: ["latest"], os-faas-80: ["rhel80-large-go-1-20"] }
matrix_spec: { version: ["latest"], os-faas-80: ["rhel87-large-go-1-20"] }
display_name: "FaaS ${version} ${os-faas-80}"
tasks:
- test-aws-lambda-task-group

- name: testgcpkms-variant
display_name: "GCP KMS"
run_on:
- debian11-small
blink1073 marked this conversation as resolved.
Show resolved Hide resolved
- rhel8.7-small
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
Expand All @@ -2779,7 +2780,7 @@ buildvariants:
- name: testawskms-variant
display_name: "AWS KMS"
run_on:
- debian11-small
- rhel8.7-small
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
Expand All @@ -2789,7 +2790,7 @@ buildvariants:
- name: testazurekms-variant
display_name: "AZURE KMS"
run_on:
- debian11-small
- rhel8.7-small
expansions:
GO_DIST: "/opt/golang/go1.20"
tasks:
Expand Down
4 changes: 2 additions & 2 deletions .evergreen/run-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ fi
export GOROOT="${GOROOT}"
export PATH="${GOROOT}/bin:${GCC_PATH}:$GOPATH/bin:$PATH"
export PROJECT="${project}"
export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
export GOFLAGS=-mod=vendor

SSL=${SSL:-nossl}
Expand Down
Loading
Loading