test(publish-metrics): add tests for otel tracing (#2718)

packages/artillery-plugin-publish-metrics/lib/open-telemetry/exporters.js

@@ -43,6 +43,10 @@ const traceExporters = {
   zipkin(options) {
     const { ZipkinExporter } = require('@opentelemetry/exporter-zipkin');
     return new ZipkinExporter(options);
+  },
+  __test(options) {
+    const { FileSpanExporter } = require('./file-span-exporter');
+    return new FileSpanExporter(options);
   }
 };
 

packages/artillery-plugin-publish-metrics/lib/open-telemetry/file-span-exporter.js

@@ -0,0 +1,73 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { ConsoleSpanExporter } = require('@opentelemetry/sdk-trace-base');
+const { ExportResultCode } = require('@opentelemetry/core');
+
+// We extend ConsoleSpanExporter as the logic is almost the same, we just need to write to a file instead of log to console
+class FileSpanExporter extends ConsoleSpanExporter {
+  constructor(opts) {
+    super();
+    this.filePath = this.setOutputPath(opts.output);
+
+    // We create the file in the main thread and then append to it in the worker threads
+    if (typeof process.env.LOCAL_WORKER_ID === 'undefined') {
+      // We write the '[' here to open an array in the file, so we can append spans to it
+      fs.writeFileSync(this.filePath, '[\n', { flag: 'w' });
+    }
+  }
+
+  _sendSpans(spans, done) {
+    const spansToExport = spans.map((span) =>
+      JSON.stringify(this._exportInfo(span))
+    );
+    if (spansToExport.length > 0) {
+      fs.writeFileSync(this.filePath, spansToExport.join(',\n') + ',', {
+        flag: 'a'
+      }); // TODO fix trailing coma
+    }
+    if (done) {
+      return done({ code: ExportResultCode.SUCCESS });
+    }
+  }
+
+  shutdown() {
+    this._sendSpans([]);
+    this.forceFlush();
+    if (typeof process.env.LOCAL_WORKER_ID === 'undefined') {
+      try {
+        // Removing the trailing comma and closing the array
+        const data =
+          fs.readFileSync(this.filePath, 'utf8').slice(0, -1) + '\n]';
+        fs.writeFileSync(this.filePath, data, { flag: 'w' });
+        console.log('File updated successfully.');
+      } catch (err) {
+        console.error('FileSpanExporter: Error updating file:');
+        throw err;
+      }
+    }
+  }
+
+  setOutputPath(output) {
+    const defaultFileName = `otel-spans-${global.artillery.testRunId}.json`;
+    const defaultOutputPath = path.resolve(process.cwd(), defaultFileName);
+    if (!output) {
+      return defaultOutputPath;
+    }
+
+    const isFile = path.extname(output);
+    const exists = isFile
+      ? fs.existsSync(path.dirname(output))
+      : fs.existsSync(output);
+
+    if (!exists) {
+      throw new Error(`FileSpanExporter: Path '${output}' does not exist`);
+    }
+    return isFile ? output : path.resolve(output, defaultFileName);
+  }
+}
+
+module.exports = {
+  FileSpanExporter
+};

packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js

@@ -57,6 +57,10 @@ class OTelTraceConfig {
       }
     }
 
+    if (this.config.__outputPath) {
+      this.exporterOpts.output = this.config.__outputPath;
+    }
+
     this.exporter = traceExporters[this.config.exporter || 'otlp-http'](
       this.exporterOpts
     );

packages/artillery-plugin-publish-metrics/package.json

@@ -51,6 +51,7 @@
   },
   "devDependencies": {
     "shelljs": "^0.8.4",
-    "tap": "^19.0.2"
+    "tap": "^19.0.2",
+    "zx": "^4.3.0"
   }
 }

packages/artillery/package.json

@@ -44,7 +44,7 @@
   },
   "scripts": {
     "test:unit": "tap --timeout=420 test/unit/*.test.js",
-    "test:acceptance": "tap --timeout=420 test/cli/*.test.js && bash test/lib/run.sh",
+    "test:acceptance": "tap --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --timeout=420 test/publish-metrics/**/*.test.js",
     "test": " npm run test:unit && npm run test:acceptance",
     "test:windows": "npm run test:unit && tap --timeout=420 test/cli/*.test.js",
     "test:aws": "tap --timeout=4200 test/cloud-e2e/**/*.test.js",

packages/artillery/test/publish-metrics/fixtures/flow.js

@@ -0,0 +1,55 @@
+'use strict';
+
+async function simpleCheck(page, userContext, events, test) {
+  await test.step('Go to Artillery', async () => {
+    const requestPromise = page.waitForRequest('https://artillery.io/');
+    await page.goto('https://artillery.io/');
+    const req = await requestPromise;
+  });
+  await test.step('Go to docs', async () => {
+    const docs = await page.getByRole('link', { name: 'Docs' });
+    await docs.click();
+    await page.waitForURL('https://www.artillery.io/docs');
+  });
+
+  await test.step('Go to core concepts', async () => {
+    await page
+      .getByRole('link', {
+        name: 'Review core concepts'
+      })
+      .click();
+
+    await page.waitForURL(
+      'https://www.artillery.io/docs/get-started/core-concepts'
+    );
+  });
+}
+
+async function simpleError(page, userContext, events, test) {
+  await test.step('Go to Artillery', async () => {
+    const requestPromise = page.waitForRequest('https://artillery.io/');
+    await page.goto('https://artillery.io/');
+    const req = await requestPromise;
+  });
+  await test.step('Go to docs', async () => {
+    const docs = await page.getByRole('link', { name: 'Docs' });
+    await docs.click();
+    await page.waitForURL('https://www.artillery.io/docs');
+  });
+
+  await test.step('Go to core concepts', async () => {
+    await page
+      .getByRole('link', {
+        name: 'Non-existent link'
+      })
+      .click();
+    await page.waitForURL(
+      'https://www.artillery.io/docs/get-started/core-concepts'
+    );
+  });
+}
+
+module.exports = {
+  simpleCheck,
+  simpleError
+};

packages/artillery/test/publish-metrics/fixtures/helpers.js

@@ -0,0 +1,41 @@
+'use strict';
+
+function getTestId(outputString) {
+  const regex = /Test run id: \S+/;
+  const match = outputString.match(regex);
+  return match[0].replace('Test run id: ', '');
+}
+
+function setDynamicHTTPTraceExpectations(expectedOutcome) {
+  if (expectedOutcome.errors) {
+    expectedOutcome.reqSpansWithError = expectedOutcome.reqSpansWithErrorPerVu
+      ? expectedOutcome.reqSpansWithErrorPerVu * expectedOutcome.vus
+      : 0;
+  }
+  expectedOutcome.spansPerVu = 1 + expectedOutcome.reqSpansPerVu;
+  expectedOutcome.reqSpans =
+    expectedOutcome.vus * expectedOutcome.reqSpansPerVu;
+  expectedOutcome.req = expectedOutcome.vus * expectedOutcome.reqPerVu;
+  expectedOutcome.totalSpans = expectedOutcome.vus * expectedOutcome.spansPerVu;
+  return expectedOutcome;
+}
+
+function setDynamicPlaywrightTraceExpectations(expectedOutcome) {
+  expectedOutcome.spansPerVu =
+    1 + expectedOutcome.pageSpansPerVu + (expectedOutcome.stepSpansPerVu || 0); // 1 represents the root scenario/VU span
+  expectedOutcome.pageSpans =
+    expectedOutcome.vus * expectedOutcome.pageSpansPerVu;
+  expectedOutcome.totalSpans = expectedOutcome.vus * expectedOutcome.spansPerVu;
+
+  if (expectedOutcome.stepSpansPerVu) {
+    expectedOutcome.stepSpans =
+      expectedOutcome.vus * expectedOutcome.stepSpansPerVu;
+  }
+  return expectedOutcome;
+}
+
+module.exports = {
+  getTestId,
+  setDynamicHTTPTraceExpectations,
+  setDynamicPlaywrightTraceExpectations
+};

packages/artillery/test/publish-metrics/fixtures/http-trace.yml

@@ -0,0 +1,26 @@
+config:
+  target: "http://asciiart.artillery.io:8080"
+  phases:
+    - duration: 2
+      arrivalRate: 2
+  plugins:
+    publish-metrics:
+      - type: "open-telemetry"
+        traces:
+          useRequestNames: true
+          replaceSpanNameRegex:
+            - pattern: "/armadillo"
+              as: "bombolini"
+          exporter: "__test"
+
+scenarios:
+  - name: "trace-http-test"
+    flow:
+      - get:
+          url: "/dino"
+          name: "dino"
+      - get:
+          url: "/pony"
+      - get:
+          url: "/armadillo"
+          name: "armadillo"

packages/artillery/test/publish-metrics/fixtures/playwright-trace.yml

@@ -0,0 +1,27 @@
+config:
+  target: "https://www.artillery.io"
+  phases:
+    - duration: 2
+      arrivalRate: 2
+  engines:
+    playwright:
+      extendedMetrics: true
+  processor: "../fixtures/flow.js"
+  plugins:
+    publish-metrics:
+      - type: "open-telemetry"
+        traces:
+          replaceSpanNameRegex:
+            - pattern: https://www.artillery.io/docs/get-started/core-concepts
+              as: core_concepts
+            - pattern: https://www.artillery.io/docs
+              as: docs_main
+          exporter: "__test"
+          attributes:
+            environment: 'test'
+            tool: 'Artillery'
+
+scenarios:
+  - engine: playwright
+    name: "trace-playwright-test"
+    testFunction: "simpleCheck"