ci(test): run aws tests against published versions (#2807)

.github/workflows/npm-publish-all-packages-canary.yml

@@ -42,6 +42,8 @@ jobs:
       packages: write
     env:
       NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+    outputs:
+      ARTILLERY_VERSION: ${{ steps.get-artillery-version.outputs.ARTILLERY_VERSION }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
@@ -65,8 +67,27 @@ jobs:
       - run: npm -w artillery-plugin-fake-data publish --tag canary
       - run: npm -w artillery-plugin-slack publish --tag canary
       - run: npm -w artillery publish --tag canary
+      - id: get-artillery-version
+        run: |
+          ARTILLERY_VERSION=$(node -e "console.log(require('./packages/artillery/package.json').version)")
+          echo "ARTILLERY_VERSION=$ARTILLERY_VERSION" >> $GITHUB_OUTPUT
       # Skytrace is a Typescript Package and needs to install -> build -> publish
       - run: npm install -w skytrace --ignore-scripts
       - run: npm run build -w skytrace
       - run: npm -w skytrace publish --tag canary
-      - run: npm -w artillery-plugin-memory-inspector publish --tag canary
+      - run: npm -w artillery-plugin-memory-inspector publish --tag canary
+
+  run-distributed-tests:
+    uses: ./.github/workflows/run-distributed-tests.yml
+    needs: build
+    with:
+      ARTILLERY_VERSION_OVERRIDE: ${{ needs.build.outputs.ARTILLERY_VERSION }}
+    permissions:
+      contents: read
+      id-token: write
+    secrets:
+      ARTILLERY_CLOUD_ENDPOINT_TEST: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }}
+      ARTILLERY_CLOUD_API_KEY_TEST: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }}
+      DD_TESTS_API_KEY: ${{ secrets.DD_TESTS_API_KEY }}
+      DD_TESTS_APP_KEY: ${{ secrets.DD_TESTS_APP_KEY }}
+      AWS_TEST_EXECUTION_ROLE_ARN_TEST5: ${{ secrets.AWS_TEST_EXECUTION_ROLE_ARN_TEST5 }}

.github/workflows/npm-publish-all-packages.yml

@@ -32,6 +32,8 @@ jobs:
     permissions:
       contents: read
       packages: write
+    outputs:
+      ARTILLERY_VERSION: ${{ steps.get-artillery-version.outputs.ARTILLERY_VERSION }}
     env:
       NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
       # this list should be kept in sync with create-release-pr.yml
@@ -67,6 +69,10 @@ jobs:
       - run: npm -w artillery-plugin-fake-data publish
       - run: npm -w artillery-plugin-slack publish
       - run: npm -w artillery publish
+      - id: get-artillery-version
+        run: |
+          ARTILLERY_VERSION=$(node -e "console.log(require('./packages/artillery/package.json').version)")
+          echo "ARTILLERY_VERSION=$ARTILLERY_VERSION" >> $GITHUB_OUTPUT
       # # Skytrace is a Typescript Package and needs to install -> build -> publish
       # - run: npm install -w skytrace --ignore-scripts
       # - run: npm run build -w skytrace
@@ -82,4 +88,19 @@ jobs:
       COMMIT_SHA: ${{ github.sha }}
     secrets:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+  run-distributed-tests:
+    uses: ./.github/workflows/run-distributed-tests.yml
+    needs: publish-packages-to-npm
+    with:
+      ARTILLERY_VERSION_OVERRIDE: ${{ needs.publish-packages-to-npm.outputs.ARTILLERY_VERSION }}
+    permissions:
+      contents: read
+      id-token: write
+    secrets:
+      ARTILLERY_CLOUD_ENDPOINT_TEST: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }}
+      ARTILLERY_CLOUD_API_KEY_TEST: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }}
+      DD_TESTS_API_KEY: ${{ secrets.DD_TESTS_API_KEY }}
+      DD_TESTS_APP_KEY: ${{ secrets.DD_TESTS_APP_KEY }}
+      AWS_TEST_EXECUTION_ROLE_ARN_TEST5: ${{ secrets.AWS_TEST_EXECUTION_ROLE_ARN_TEST5 }}

.github/workflows/run-aws-tests-on-pr.yml

@@ -1,4 +1,4 @@
-name: Run AWS tests
+name: Run AWS tests (on PR)
 
 on:
   pull_request_target:

.github/workflows/run-distributed-tests.yml

@@ -1,4 +1,4 @@
-name: Run AWS tests
+name: Run distributed tests
 
 on:
   workflow_call:
@@ -58,7 +58,6 @@ jobs:
       contents: read
       id-token: write
     env:
-      ECR_IMAGE_VERSION: ${{ inputs.COMMIT_SHA || github.sha }}
       ARTILLERY_CLOUD_ENDPOINT: ${{ secrets.ARTILLERY_CLOUD_ENDPOINT_TEST }}
       ARTILLERY_CLOUD_API_KEY: ${{ secrets.ARTILLERY_CLOUD_API_KEY_TEST }}
       DD_TESTS_API_KEY: ${{ secrets.DD_TESTS_API_KEY }}
@@ -84,6 +83,17 @@ jobs:
           node-version: 18.x
       - run: npm install
       - run: npm run build
+      - name: Install Specific Artillery Version if needed
+        if: ${{ inputs.ARTILLERY_VERSION_OVERRIDE || false }}
+        run: mkdir __artillery__ && cd __artillery__ && npm init -y && npm install artillery@${{ inputs.ARTILLERY_VERSION_OVERRIDE }}
+      - name: Set A9_PATH
+        if: ${{ inputs.ARTILLERY_VERSION_OVERRIDE || false }}
+        run: echo "A9_PATH=${{ github.workspace }}/__artillery__/node_modules/.bin/artillery" >> $GITHUB_ENV
+      - name: Set ECR Image Version if needed
+        if: ${{ inputs.COMMIT_SHA }}
+        run: |
+          echo "ECR_IMAGE_VERSION=${{ inputs.COMMIT_SHA }}" >> $GITHUB_ENV
+          echo "LAMBDA_IMAGE_VERSION=${{ inputs.COMMIT_SHA }}" >> $GITHUB_ENV
       # runs the single test file from `package` workspace in the `file`, as defined in the matrix output
       - run: npm run test:aws:ci --workspace ${{fromJson(needs.generate-test-matrix.outputs.matrix).namesToFiles[matrix.testName].package }} -- --files ${{ fromJson(needs.generate-test-matrix.outputs.matrix).namesToFiles[matrix.testName].file }}
         env:

packages/artillery-engine-playwright/test/fargate.aws.js

@@ -1,4 +1,4 @@
-const { test, afterEach, beforeEach } = require('tap');
+const { test, afterEach, beforeEach, before } = require('tap');
 const { $ } = require('zx');
 const { getTestTags } = require('../../artillery/test/cli/_helpers.js');
 const fs = require('fs');
@@ -15,9 +15,14 @@ afterEach(async () => {
   fs.unlinkSync(playwrightOutput);
 });
 
+const A9_PATH = process.env.A9_PATH || '../artillery/bin/run';
+before(async () => {
+  await $`${A9_PATH} -V`;
+});
+
 test('playwright typescript test works and reports data', async (t) => {
   const output =
-    await $`../artillery/bin/run run:fargate ./test/fixtures/pw-acceptance-ts.yml --output ${playwrightOutput} --tags ${tags} --record`;
+    await $`${A9_PATH} run:fargate ./test/fixtures/pw-acceptance-ts.yml --output ${playwrightOutput} --tags ${tags} --record`;
 
   t.equal(
     output.exitCode,
@@ -122,7 +127,7 @@ test('playwright typescript test fails and has correct vu count when expectation
   });
 
   try {
-    await $`../artillery/bin/run run:fargate ./test/fixtures/pw-acceptance-ts.yml --output ${playwrightOutput} --overrides ${scenarioOverride} --tags ${tags} --record`;
+    await $`${A9_PATH} run:fargate ./test/fixtures/pw-acceptance-ts.yml --output ${playwrightOutput} --overrides ${scenarioOverride} --tags ${tags} --record`;
     t.fail(`Test "${t.name}" - Should have had non-zero exit code.`);
   } catch (output) {
     t.equal(

packages/artillery/test/cloud-e2e/fargate/bom.test.js

@@ -7,10 +7,10 @@ const {
   execute
 } = require('../../cli/_helpers.js');
 
-const A9 = process.env.A9 || 'artillery';
+const A9_PATH = process.env.A9_PATH || 'artillery';
 
 before(async () => {
-  await $`${A9} -V`;
+  await $`${A9_PATH} -V`;
 });
 
 //NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
@@ -54,7 +54,7 @@ test('Run mixed-hierarchy', async (t) => {
   const configPath = `${__dirname}/fixtures/mixed-hierarchy/config/config.yml`;
 
   const output =
-    await $`${A9} run-fargate ${scenarioPath} --config ${configPath} -e main --record --tags ${baseTags} --output ${reportFilePath}`;
+    await $`${A9_PATH} run-fargate ${scenarioPath} --config ${configPath} -e main --record --tags ${baseTags} --output ${reportFilePath}`;
 
   const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));
 

packages/artillery/test/cloud-e2e/fargate/cw-adot.test.js

@@ -1,16 +1,16 @@
 'use strict';
 
-const { test, afterEach, beforeEach } = require('tap');
+const { test, afterEach, beforeEach, before } = require('tap');
 const { $ } = require('zx');
 const fs = require('fs');
 const {
   generateTmpReportPath,
   deleteFile,
   getTestTags
 } = require('../../cli/_helpers.js');
-
 const { getTestId, getXRayTraces } = require('./fixtures/adot/helpers.js');
 
+const A9_PATH = process.env.A9_PATH || 'artillery';
 // NOTE: This test reports to Artillery Dashboard to dogfood and improve visibility
 const baseTags = getTestTags(['type:acceptance']);
 
@@ -23,6 +23,10 @@ afterEach(async (t) => {
   deleteFile(reportFilePath);
 });
 
+before(async () => {
+  await $`${A9_PATH} -V`;
+});
+
 test('traces succesfully arrive to cloudwatch', async (t) => {
   // Arrange:
   const expectedTotalSpans = 28; // 4 VUs * (1 scenario root span + 3 pageSpans + 3 stepSpans )
@@ -36,7 +40,7 @@ test('traces succesfully arrive to cloudwatch', async (t) => {
 
   // Act:
   const output =
-    await $`artillery run-fargate ${__dirname}/fixtures/adot/adot-cloudwatch.yml --record --tags ${baseTags} --output ${reportFilePath}`;
+    await $`${A9_PATH} run-fargate ${__dirname}/fixtures/adot/adot-cloudwatch.yml --record --tags ${baseTags} --output ${reportFilePath}`;
 
   const testId = getTestId(output.stdout);
   const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));

packages/artillery/test/cloud-e2e/fargate/dd-adot.test.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { test, afterEach, beforeEach } = require('tap');
+const { test, afterEach, beforeEach, before } = require('tap');
 const { $ } = require('zx');
 const fs = require('fs');
 const {
@@ -11,6 +11,7 @@ const {
 
 const { getDatadogSpans, getTestId } = require('./fixtures/adot/helpers.js');
 
+const A9_PATH = process.env.A9_PATH || 'artillery';
 //NOTE: This test reports to Artillery Dashboard to dogfood and improve visibility
 const baseTags = getTestTags(['type:acceptance']);
 
@@ -23,6 +24,10 @@ afterEach(async (t) => {
   deleteFile(reportFilePath);
 });
 
+before(async () => {
+  await $`${A9_PATH} -V`;
+});
+
 test('traces succesfully arrive to datadog', async (t) => {
   // Arrange:
   const apiKey = process.env.DD_TESTS_API_KEY;
@@ -43,7 +48,7 @@ test('traces succesfully arrive to datadog', async (t) => {
 
   // Act:
   const output =
-    await $`artillery run-fargate ${__dirname}/fixtures/adot/adot-dd-pass.yml --record --tags ${baseTags} --output ${reportFilePath}`;
+    await $`${A9_PATH} run-fargate ${__dirname}/fixtures/adot/adot-dd-pass.yml --record --tags ${baseTags} --output ${reportFilePath}`;
 
   const testId = getTestId(output.stdout);
   const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8'));

packages/artillery/test/cloud-e2e/fargate/ensure-plugin.test.js

@@ -4,10 +4,10 @@ const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../cli/_helpers.js');
 
-const A9 = process.env.A9 || 'artillery';
+const A9_PATH = process.env.A9_PATH || 'artillery';
 
 before(async () => {
-  await $`${A9} -V`;
+  await $`${A9_PATH} -V`;
 });
 
 //NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
@@ -19,7 +19,7 @@ beforeEach(async (t) => {
 
 test('Run uses ensure', async (t) => {
   try {
-    await $`${A9} run:fargate ${__dirname}/fixtures/uses-ensure/with-ensure.yaml --record --tags ${baseTags} --output ${reportFilePath} --count 15`;
+    await $`${A9_PATH} run:fargate ${__dirname}/fixtures/uses-ensure/with-ensure.yaml --record --tags ${baseTags} --output ${reportFilePath} --count 15`;
     t.fail(`Test "${t.name}" - Should have had non-zero exit code.`);
   } catch (output) {
     t.equal(output.exitCode, 1, 'CLI Exit Code should be 1');

packages/artillery/test/cloud-e2e/fargate/expect-plugin.test.js

@@ -4,10 +4,10 @@ const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../cli/_helpers.js');
 
-const A9 = process.env.A9 || 'artillery';
+const A9_PATH = process.env.A9_PATH || 'artillery';
 
 before(async () => {
-  await $`${A9} -V`;
+  await $`${A9_PATH} -V`;
 });
 
 //NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
@@ -19,7 +19,7 @@ beforeEach(async (t) => {
 
 test('CLI should exit with non-zero exit code when there are failed expectations in workers', async (t) => {
   try {
-    await $`${A9} run-fargate ${__dirname}/fixtures/cli-exit-conditions/with-expect.yml --record --tags ${baseTags} --output ${reportFilePath} --count 2`;
+    await $`${A9_PATH} run-fargate ${__dirname}/fixtures/cli-exit-conditions/with-expect.yml --record --tags ${baseTags} --output ${reportFilePath} --count 2`;
     t.fail(`Test "${t.name}" - Should have had non-zero exit code.`);
   } catch (output) {
     t.equal(output.exitCode, 6, 'CLI Exit Code should be 6');
@@ -42,7 +42,7 @@ test('Ensure (with new interface) should still run when workers exit from expect
   //Note: this test uses new ensure plugin interface (config.plugins.ensure) to test that indirectly
 
   try {
-    await $`${A9} run:fargate ${__dirname}/fixtures/cli-exit-conditions/with-expect-ensure.yml --record --tags ${baseTags} --output ${reportFilePath} --count 2`;
+    await $`${A9_PATH} run:fargate ${__dirname}/fixtures/cli-exit-conditions/with-expect-ensure.yml --record --tags ${baseTags} --output ${reportFilePath} --count 2`;
     t.fail(`Test "${t.name}" - Should have had non-zero exit code.`);
   } catch (output) {
     t.equal(output.exitCode, 1, 'CLI Exit Code should be 1');

packages/artillery/test/cloud-e2e/fargate/memory.test.js

@@ -2,18 +2,18 @@ const { test, before, beforeEach } = require('tap');
 const { $ } = require('zx');
 const { generateTmpReportPath, getTestTags } = require('../../cli/_helpers.js');
 
-const A9 = process.env.A9 || 'artillery';
+const A9_PATH = process.env.A9_PATH || 'artillery';
 
 before(async () => {
-  await $`${A9} -V`;
+  await $`${A9_PATH} -V`;
 });
 
 //NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
 const baseTags = getTestTags(['type:acceptance']);
 
 test('Fargate should exit with error code when workers run out of memory', async (t) => {
   try {
-    await $`${A9} run-fargate ${__dirname}/fixtures/memory-hog/memory-hog.yml --record --tags ${baseTags},should_fail:true --region us-east-1`;
+    await $`${A9_PATH} run-fargate ${__dirname}/fixtures/memory-hog/memory-hog.yml --record --tags ${baseTags},should_fail:true --region us-east-1`;
     t.fail(`Test "${t.name}" - Should have had non-zero exit code.`);
   } catch (output) {
     t.equal(output.exitCode, 6, 'CLI Exit Code should be 6');
@@ -25,7 +25,7 @@ test('Fargate should exit with error code when workers run out of memory', async
 
 test('Fargate should not run out of memory when cpu and memory is increased via launch config', async (t) => {
   const output =
-    await $`${A9} run-fargate ${__dirname}/fixtures/memory-hog/memory-hog.yml --record --tags ${baseTags},should_fail:false --region us-east-1 --launch-config '{"cpu":"4096", "memory":"12288"}'`;
+    await $`${A9_PATH} run-fargate ${__dirname}/fixtures/memory-hog/memory-hog.yml --record --tags ${baseTags},should_fail:false --region us-east-1 --launch-config '{"cpu":"4096", "memory":"12288"}'`;
 
   t.equal(output.exitCode, 0, 'CLI Exit Code should be 0');
   t.match(output, /summary report/i, 'print summary report');
@@ -34,7 +34,7 @@ test('Fargate should not run out of memory when cpu and memory is increased via
 
 test('Fargate should not run out of memory when cpu and memory is increased via flags', async (t) => {
   const output =
-    await $`${A9} run-fargate ${__dirname}/fixtures/memory-hog/memory-hog.yml --record --tags ${baseTags},should_fail:false --region us-east-1 --cpu 4 --memory 12`;
+    await $`${A9_PATH} run-fargate ${__dirname}/fixtures/memory-hog/memory-hog.yml --record --tags ${baseTags},should_fail:false --region us-east-1 --cpu 4 --memory 12`;
 
   t.equal(output.exitCode, 0, 'CLI Exit Code should be 0');
   t.match(output, /summary report/i, 'print summary report');

packages/artillery/test/cloud-e2e/fargate/misc.test.js

@@ -4,10 +4,10 @@ const chalk = require('chalk');
 const fs = require('fs');
 const { generateTmpReportPath, getTestTags } = require('../../cli/_helpers.js');
 
-const A9 = process.env.A9 || 'artillery';
+const A9_PATH = process.env.A9_PATH || 'artillery';
 
 before(async () => {
-  await $`${A9} -V`;
+  await $`${A9_PATH} -V`;
 });
 
 //NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility
@@ -30,7 +30,7 @@ test('Kitchen Sink Test - multiple features together', async (t) => {
   };
 
   const output =
-    await $`${A9} run-fargate ${scenarioPath} --output ${reportFilePath} --dotenv ${dotEnvPath} --record --tags ${baseTags} --count 2 --region us-east-2 --spot --launch-config ${JSON.stringify(
+    await $`${A9_PATH} run-fargate ${scenarioPath} --output ${reportFilePath} --dotenv ${dotEnvPath} --record --tags ${baseTags} --count 2 --region us-east-2 --spot --launch-config ${JSON.stringify(
       launchConfig
     )}`;
 
@@ -83,7 +83,7 @@ test('Run lots-of-output', async (t) => {
   $.verbose = false; // we don't want megabytes of output on the console
 
   const output =
-    await $`${A9} run:fargate ${__dirname}/fixtures/large-output/lots-of-output.yml --record --tags ${baseTags}`;
+    await $`${A9_PATH} run:fargate ${__dirname}/fixtures/large-output/lots-of-output.yml --record --tags ${baseTags}`;
 
   t.equal(output.exitCode, 0, 'CLI Exit Code should be 0');