diff --git a/package-lock.json b/package-lock.json index 947ca3cc5f..bdc45b8c95 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23432,7 +23432,8 @@ }, "devDependencies": { "shelljs": "^0.8.4", - "tap": "^19.0.2" + "tap": "^19.0.2", + "zx": "^4.3.0" } }, "packages/artillery-plugin-publish-metrics/node_modules/@aws-crypto/ie11-detection": { diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/exporters.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/exporters.js index c17f31d6b7..73c28e9095 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/exporters.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/exporters.js @@ -43,6 +43,10 @@ const traceExporters = { zipkin(options) { const { ZipkinExporter } = require('@opentelemetry/exporter-zipkin'); return new ZipkinExporter(options); + }, + __test(options) { + const { FileSpanExporter } = require('./file-span-exporter'); + return new FileSpanExporter(options); } }; diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/file-span-exporter.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/file-span-exporter.js new file mode 100644 index 0000000000..906d7fbd60 --- /dev/null +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/file-span-exporter.js @@ -0,0 +1,73 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { ConsoleSpanExporter } = require('@opentelemetry/sdk-trace-base'); +const { ExportResultCode } = require('@opentelemetry/core'); + +// We extend ConsoleSpanExporter as the logic is almost the same, we just need to write to a file instead of log to console +class FileSpanExporter extends ConsoleSpanExporter { + constructor(opts) { + super(); + this.filePath = this.setOutputPath(opts.output); + + // We create the file in the main thread and then append to it in the worker threads + if (typeof process.env.LOCAL_WORKER_ID === 'undefined') { + // We write the '[' here to open an array in the file, so we can append spans to it + fs.writeFileSync(this.filePath, '[\n', { flag: 'w' }); + } + } + + _sendSpans(spans, done) { + const spansToExport = spans.map((span) => + JSON.stringify(this._exportInfo(span)) + ); + if (spansToExport.length > 0) { + fs.writeFileSync(this.filePath, spansToExport.join(',\n') + ',', { + flag: 'a' + }); // TODO fix trailing coma + } + if (done) { + return done({ code: ExportResultCode.SUCCESS }); + } + } + + shutdown() { + this._sendSpans([]); + this.forceFlush(); + if (typeof process.env.LOCAL_WORKER_ID === 'undefined') { + try { + // Removing the trailing comma and closing the array + const data = + fs.readFileSync(this.filePath, 'utf8').slice(0, -1) + '\n]'; + fs.writeFileSync(this.filePath, data, { flag: 'w' }); + console.log('File updated successfully.'); + } catch (err) { + console.error('FileSpanExporter: Error updating file:'); + throw err; + } + } + } + + setOutputPath(output) { + const defaultFileName = `otel-spans-${global.artillery.testRunId}.json`; + const defaultOutputPath = path.resolve(process.cwd(), defaultFileName); + if (!output) { + return defaultOutputPath; + } + + const isFile = path.extname(output); + const exists = isFile + ? fs.existsSync(path.dirname(output)) + : fs.existsSync(output); + + if (!exists) { + throw new Error(`FileSpanExporter: Path '${output}' does not exist`); + } + return isFile ? output : path.resolve(output, defaultFileName); + } +} + +module.exports = { + FileSpanExporter +}; diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js index 7674a143af..53d656435c 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js @@ -57,6 +57,10 @@ class OTelTraceConfig { } } + if (this.config.__outputPath) { + this.exporterOpts.output = this.config.__outputPath; + } + this.exporter = traceExporters[this.config.exporter || 'otlp-http']( this.exporterOpts ); diff --git a/packages/artillery-plugin-publish-metrics/package.json b/packages/artillery-plugin-publish-metrics/package.json index c3dca75725..1be2f7cfa0 100644 --- a/packages/artillery-plugin-publish-metrics/package.json +++ b/packages/artillery-plugin-publish-metrics/package.json @@ -51,6 +51,7 @@ }, "devDependencies": { "shelljs": "^0.8.4", - "tap": "^19.0.2" + "tap": "^19.0.2", + "zx": "^4.3.0" } } diff --git a/packages/artillery/package.json b/packages/artillery/package.json index bbad94550c..d50790262d 100644 --- a/packages/artillery/package.json +++ b/packages/artillery/package.json @@ -44,7 +44,7 @@ }, "scripts": { "test:unit": "tap --timeout=420 test/unit/*.test.js", - "test:acceptance": "tap --timeout=420 test/cli/*.test.js && bash test/lib/run.sh", + "test:acceptance": "tap --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --timeout=420 test/publish-metrics/**/*.test.js", "test": " npm run test:unit && npm run test:acceptance", "test:windows": "npm run test:unit && tap --timeout=420 test/cli/*.test.js", "test:aws": "tap --timeout=4200 test/cloud-e2e/**/*.test.js", diff --git a/packages/artillery/test/publish-metrics/fixtures/flow.js b/packages/artillery/test/publish-metrics/fixtures/flow.js new file mode 100644 index 0000000000..9f53748e3b --- /dev/null +++ b/packages/artillery/test/publish-metrics/fixtures/flow.js @@ -0,0 +1,55 @@ +'use strict'; + +async function simpleCheck(page, userContext, events, test) { + await test.step('Go to Artillery', async () => { + const requestPromise = page.waitForRequest('https://artillery.io/'); + await page.goto('https://artillery.io/'); + const req = await requestPromise; + }); + await test.step('Go to docs', async () => { + const docs = await page.getByRole('link', { name: 'Docs' }); + await docs.click(); + await page.waitForURL('https://www.artillery.io/docs'); + }); + + await test.step('Go to core concepts', async () => { + await page + .getByRole('link', { + name: 'Review core concepts' + }) + .click(); + + await page.waitForURL( + 'https://www.artillery.io/docs/get-started/core-concepts' + ); + }); +} + +async function simpleError(page, userContext, events, test) { + await test.step('Go to Artillery', async () => { + const requestPromise = page.waitForRequest('https://artillery.io/'); + await page.goto('https://artillery.io/'); + const req = await requestPromise; + }); + await test.step('Go to docs', async () => { + const docs = await page.getByRole('link', { name: 'Docs' }); + await docs.click(); + await page.waitForURL('https://www.artillery.io/docs'); + }); + + await test.step('Go to core concepts', async () => { + await page + .getByRole('link', { + name: 'Non-existent link' + }) + .click(); + await page.waitForURL( + 'https://www.artillery.io/docs/get-started/core-concepts' + ); + }); +} + +module.exports = { + simpleCheck, + simpleError +}; diff --git a/packages/artillery/test/publish-metrics/fixtures/helpers.js b/packages/artillery/test/publish-metrics/fixtures/helpers.js new file mode 100644 index 0000000000..ad8b2e189b --- /dev/null +++ b/packages/artillery/test/publish-metrics/fixtures/helpers.js @@ -0,0 +1,41 @@ +'use strict'; + +function getTestId(outputString) { + const regex = /Test run id: \S+/; + const match = outputString.match(regex); + return match[0].replace('Test run id: ', ''); +} + +function setDynamicHTTPTraceExpectations(expectedOutcome) { + if (expectedOutcome.errors) { + expectedOutcome.reqSpansWithError = expectedOutcome.reqSpansWithErrorPerVu + ? expectedOutcome.reqSpansWithErrorPerVu * expectedOutcome.vus + : 0; + } + expectedOutcome.spansPerVu = 1 + expectedOutcome.reqSpansPerVu; + expectedOutcome.reqSpans = + expectedOutcome.vus * expectedOutcome.reqSpansPerVu; + expectedOutcome.req = expectedOutcome.vus * expectedOutcome.reqPerVu; + expectedOutcome.totalSpans = expectedOutcome.vus * expectedOutcome.spansPerVu; + return expectedOutcome; +} + +function setDynamicPlaywrightTraceExpectations(expectedOutcome) { + expectedOutcome.spansPerVu = + 1 + expectedOutcome.pageSpansPerVu + (expectedOutcome.stepSpansPerVu || 0); // 1 represents the root scenario/VU span + expectedOutcome.pageSpans = + expectedOutcome.vus * expectedOutcome.pageSpansPerVu; + expectedOutcome.totalSpans = expectedOutcome.vus * expectedOutcome.spansPerVu; + + if (expectedOutcome.stepSpansPerVu) { + expectedOutcome.stepSpans = + expectedOutcome.vus * expectedOutcome.stepSpansPerVu; + } + return expectedOutcome; +} + +module.exports = { + getTestId, + setDynamicHTTPTraceExpectations, + setDynamicPlaywrightTraceExpectations +}; diff --git a/packages/artillery/test/publish-metrics/fixtures/http-trace.yml b/packages/artillery/test/publish-metrics/fixtures/http-trace.yml new file mode 100644 index 0000000000..00f527aa93 --- /dev/null +++ b/packages/artillery/test/publish-metrics/fixtures/http-trace.yml @@ -0,0 +1,26 @@ +config: + target: "http://asciiart.artillery.io:8080" + phases: + - duration: 2 + arrivalRate: 2 + plugins: + publish-metrics: + - type: "open-telemetry" + traces: + useRequestNames: true + replaceSpanNameRegex: + - pattern: "/armadillo" + as: "bombolini" + exporter: "__test" + +scenarios: + - name: "trace-http-test" + flow: + - get: + url: "/dino" + name: "dino" + - get: + url: "/pony" + - get: + url: "/armadillo" + name: "armadillo" diff --git a/packages/artillery/test/publish-metrics/fixtures/playwright-trace.yml b/packages/artillery/test/publish-metrics/fixtures/playwright-trace.yml new file mode 100644 index 0000000000..03bc4f8a6a --- /dev/null +++ b/packages/artillery/test/publish-metrics/fixtures/playwright-trace.yml @@ -0,0 +1,27 @@ +config: + target: "https://www.artillery.io" + phases: + - duration: 2 + arrivalRate: 2 + engines: + playwright: + extendedMetrics: true + processor: "../fixtures/flow.js" + plugins: + publish-metrics: + - type: "open-telemetry" + traces: + replaceSpanNameRegex: + - pattern: https://www.artillery.io/docs/get-started/core-concepts + as: core_concepts + - pattern: https://www.artillery.io/docs + as: docs_main + exporter: "__test" + attributes: + environment: 'test' + tool: 'Artillery' + +scenarios: + - engine: playwright + name: "trace-playwright-test" + testFunction: "simpleCheck" diff --git a/packages/artillery/test/publish-metrics/tracing/http-trace-assertions.js b/packages/artillery/test/publish-metrics/tracing/http-trace-assertions.js new file mode 100644 index 0000000000..25a4a9aaf0 --- /dev/null +++ b/packages/artillery/test/publish-metrics/tracing/http-trace-assertions.js @@ -0,0 +1,327 @@ +'use strict'; + +const { getTestId } = require('../fixtures/helpers.js'); + +const requestPhasesAttrs = [ + 'dns_lookup.duration', + 'tcp_handshake.duration', + 'request.duration', + 'download.duration', + 'response.time.ms' +]; // There is also 'tls_negotiation' but it will not be present in the spans as the test does not make https requests + +const httpRequestAttrs = [ + 'http.method', + 'http.url', + 'http.scheme', + 'net.host.name' +]; + +const httpResponseAttrs = [ + 'http.status_code', + 'http.flavor', + 'http.user_agent' +]; + +/** Runs assertions for OTel Playwright tracing tests using 'tap' library. It checks that the trace data is correctly recorded, formatted and exported by the OTel plugin. + * @param {Object} t - the tap library test object + * @param {Object} testRunData - an object containing the console output of the test run, the report summary and the exported spans - `{ output, reportSummary, spans }` + * @namespace expectedOutcome + * @param {Object} expectedOutcome - an object containing the expected outcome values for the test run. + * @param {string} expectedOutcome.scenarioName - the name of the scenario + * @param {number} expectedOutcome.exitCode - the expected exit code of the test run + * @param {number} expectedOutcome.vus - the number of VUs created + * @param {number} expectedOutcome.reqPerVu - the number of requests made per VU + * @param {number} expectedOutcome.reqSpansPerVu - the number of request spans created per VU + * @param {number} [expectedOutcome.vusFailed] - the number of VUs that failed + * @param {number} [expectedOutcome.errors] - the number of errors recorded + * @param {number} [expectedOutcome.reqSpansWithErrorPerVu] - the number of request spans with errors recorded per VU + * @param {number} expectedOutcome.spansWithErrorStatus - the total number of spans with error status + * @param {Object} [expectedOutcome.userSetAttributes] - an object containing the user set attributes + * @param {Array} [expectedOutcome.spanNamesByReqName] - an array of span names to be set by the request name + * @param {Array} [expectedOutcome.spanNamesByMethod] - an array of span names to be set by the request method + * @param {Array} [expectedOutcome.spanNamesReplaced] - an array of span names to be replaced by the replaceSpanNameRegex setting + * @param {number} expectedOutcome.spansPerVu - the total number of spans created per VU - `setDynamicHTTPTraceExpectations` function can be used to calculate this value + * @param {number} expectedOutcome.reqSpans - the total number of request spans created - `setDynamicHTTPTraceExpectations` function can be used to calculate this value + * @param {number} expectedOutcome.totalSpans - the total number of spans to be created - `setDynamicHTTPTraceExpectations` function can be used to calculate this value + * @param {number} expectedOutcome.reqSpansWithError - the total number of request spans with errors recorded - `setDynamicHTTPTraceExpectations` function can be used to calculate this value + * @param {number} expectedOutcome.totalSpans - the total number of spans to be created - `setDynamicPlaywrightTraceExpectations` function can be used to calculate this value + * + * + * #### Configuration settings and functionality covered: + * - `scenarioName` config setting - sets the scenario span name + * - `useRequestNames` config setting - sets the request span names to the request name + * - `replaceSpanNameRegex` config setting - replaces the specified pattern in page span names + * - `attributes` config setting - sets the user attributes for all spans + * - default id attributes - `test_id` and `vu.uuid` attributes for all spans + * - http request and response attributes - sets the http request and response specific attributes for the request spans + * - request phases attributes - sets the request phases attributes for the request spans + * - `plugins.publish-metrics.spans.exported` metric - emits the counter metric for the number of spans exported + * - errors - errors are recorded on traces both as error events and as the error status code + * + * + * #### Configuration settings and functionality not covered - needs to be implemented in the future: + * - `sampleRate` config setting - loose percentage of spans to be sampled + * - `smartSampling` config setting - tags and exports response outliers + * + * If any new features are added that add to or change the tracing format, this is where the change should be implemented to propagate to all tests. + */ +async function runHttpTraceAssertions(t, testRunData, expectedOutcome) { + const { output, reportSummary, spans } = testRunData; + + const testId = getTestId(output.stdout); + const requestSpans = spans.filter((span) => span.attributes['http.method']); + const scenarioSpans = spans.filter((span) => !span.parentId); + + // Created VUs/traces + t.equal( + reportSummary.counters['vusers.created'], + expectedOutcome.vus, + `${expectedOutcome.vus} VUs should have been created` + ); + t.equal( + reportSummary.counters['vusers.created'], + scenarioSpans.length, + 'The number of scenario spans should match the number of VUs created' + ); + t.equal( + spans.length, + expectedOutcome.totalSpans, + `There should be ${expectedOutcome.totalSpans} spans created in total` + ); + + // Errors and failed VUs + t.equal( + output.exitCode, + expectedOutcome.exitCode, + `CLI Exit Code should be ${expectedOutcome.exitCode}` + ); + t.equal( + reportSummary.counters['vusers.failed'], + expectedOutcome.vusFailed || 0, + `${expectedOutcome.vusFailed} VUs should have failed` + ); + + const errorsReported = Object.keys(reportSummary.counters).filter( + (metricName) => metricName.startsWith('errors.') + ); + const numErrorsReported = errorsReported.reduce( + (acc, metricName) => acc + reportSummary.counters[metricName], + 0 + ); + t.equal( + numErrorsReported, + expectedOutcome.errors || 0, + `There should be ${expectedOutcome.errors} errors reported` + ); + t.equal( + spans.filter((span) => span.events[0]?.name === 'exception').length, + expectedOutcome.errors || 0, + 'Num of errors in report should match the num of spans with error exception' + ); // In http engine the only event we record is the error exception event so we can just check that event is present + + // We check the error span status separately from errors as it can be set to error even when no error is recorded, e.g. when http status code is 404 or over + t.equal( + spans.filter((span) => span.status.code === 2).length, + expectedOutcome.spansWithErrorStatus, + `${expectedOutcome.spansWithErrorStatus} spans should have the 'error' status` + ); + + if (expectedOutcome.errors || numErrorsReported) { + const errorNum = expectedOutcome.errors || numErrorsReported; + t.equal( + spans.filter( + (span) => span.events[0]?.name === 'exception' && span.status.code === 2 + ).length, + errorNum, + 'Errors should be recorded on spans as an event and status code' + ); + t.equal( + requestSpans.filter((span) => span.events[0]?.name === 'exception') + .length, + errorNum, + `${errorNum} request spans should have the error exception recorded` + ); + spans + .filter((span) => span.events[0]?.name === 'exception') + .forEach((span) => { + t.hasProps( + span.events[0].attributes, + ['exception.type', 'exception.message', 'exception.stacktrace'], + 'Every error event recorded should have the error type, message and stacktrace recorded' + ); + }); + } + + // Request level spans + t.equal( + reportSummary.counters['http.requests'], + expectedOutcome.req, + `${expectedOutcome.req} requests should have been made` + ); + + t.equal( + requestSpans.length, + expectedOutcome.reqSpans, + `There should be ${expectedOutcome.reqSpans} request spans created in total.` + ); + + // If an error happens when trying to make a request (after before request hook) resulting in request not being made, we will still have the request span for it with the error recorded on the span + // So the number of request spans will not be equal to the number of requests made + if (!expectedOutcome.errors) { + t.equal( + requestSpans.length, + reportSummary.counters['http.requests'], + 'The number of request spans should match the number of requests made' + ); + } + + Object.keys(reportSummary.counters) + .filter((counter) => { + counter.startsWith('http.codes.'); + }) + .forEach((metric) => { + const statusCode = metric.split('.')[2]; + t.equal( + requestSpans.filter( + (span) => + span.attributes['http.status_code'] && + span.attributes['http.status_code'] === statusCode + ).length, + reportSummary.counters[metric], + `The number of spans with status code ${statusCode} should match the number of requests with that status code` + ); + }); + + // Span names + t.equal( + scenarioSpans[0].name, + expectedOutcome.scenarioName, + 'The scenario span should have the name of the scenario when set' + ); + + // `useRequestNames` check + expectedOutcome.spanNamesByReqName + .filter((span) => !expectedOutcome.spanNamesReplaced.includes(span.name)) + .forEach((name) => { + t.equal( + requestSpans.filter((span) => span.name === name).length, + requestSpans.length / expectedOutcome.reqSpansPerVu, + 'When useRequestNames is set to true, the request span should have the name of the request if the name is set' + ); + }); + + expectedOutcome.spanNamesByMethod.forEach((name) => { + t.equal( + requestSpans.filter((span) => span.name === name).length, + requestSpans.length / expectedOutcome.reqSpansPerVu, + 'If useRequestNames is not set, or if no request name is provided,the request span will be named by the request method' + ); + }); + + // `replaceSpanNameRegex` check + expectedOutcome.spanNamesReplaced.forEach((name) => { + t.equal( + spans.filter((span) => span.name === name).length, + spans.length / expectedOutcome.spansPerVu, + 'replaceSpanNameRegex appropriately replaces the pattern in span name' + ); + }); + + // Proper nesting + const reqSpanNamesPerVU = expectedOutcome.spanNamesByReqName.concat( + expectedOutcome.spanNamesByMethod + ); + scenarioSpans + .map((span) => span.id) + .forEach((id) => { + const siblingRequestSpans = requestSpans.filter( + (requestSpan) => requestSpan.parentId === id + ); + t.equal( + siblingRequestSpans.length, + expectedOutcome.reqSpansPerVu, + `Each trace should have ${expectedOutcome.reqSpansPerVu} request spans` + ); + siblingRequestSpans.forEach((span) => { + t.ok( + reqSpanNamesPerVU.includes(span.name), + `Each trace should have a request span called ${span.name}` + ); + }); + }); + + // Attributes + t.equal( + spans.filter((span) => span.attributes['test_id']).length, + spans.length, + 'All spans should have the test_id attribute' + ); + t.equal( + spans.filter((span) => span.attributes['test_id'] === testId).length, + spans.length, + 'All spans should have the correct test_id attribute value' + ); + t.equal( + spans.filter((span) => span.attributes['vu.uuid']).length, + spans.length, + 'All spans should have the vu.uuid attribute' + ); + + requestSpans.forEach((span) => { + t.hasProps( + span.attributes, + httpRequestAttrs, + 'All request spans should have the http request specific attributes' + ); + }); + + // Only check for request phases and httpResponseAttrs on successful requests that have no exceptions or error status + requestSpans + .filter( + (span) => + !span.events[0] || + span.events[0].name !== 'exception' || + span.status.code !== 2 + ) + .forEach((span) => { + t.hasProps( + span.attributes, + httpResponseAttrs, + 'All successful request spans should have the http response specific attributes' + ); + + t.hasProps( + span.attributes, + requestPhasesAttrs, + 'All successful request spans should have all request phases set as attributes' + ); + }); + + Object.keys(expectedOutcome.userSetAttributes).forEach((attr) => { + t.equal( + requestSpans.filter((span) => span.attributes[attr]).length, + requestSpans.length, + 'All request should have the user set attributes' + ); + t.equal( + requestSpans.filter( + (span) => + span.attributes[attr] === expectedOutcome.userSetAttributes[attr] + ).length, + requestSpans.length, + 'Correct values should be set for all user provided attributes' + ); + }); + + // Counter metric reported + t.equal( + reportSummary.counters['plugins.publish-metrics.spans.exported'], + expectedOutcome.totalSpans, + 'The `plugins.publish-metrics.spans.exported` counter should match the total number of spans exported' + ); +} + +module.exports = { + runHttpTraceAssertions +}; diff --git a/packages/artillery/test/publish-metrics/tracing/http-trace.test.js b/packages/artillery/test/publish-metrics/tracing/http-trace.test.js new file mode 100644 index 0000000000..5754b422f1 --- /dev/null +++ b/packages/artillery/test/publish-metrics/tracing/http-trace.test.js @@ -0,0 +1,274 @@ +const { test, afterEach, beforeEach } = require('tap'); +const { $ } = require('zx'); +const fs = require('fs'); +const { generateTmpReportPath, deleteFile } = require('../../cli/_helpers.js'); + +const { setDynamicHTTPTraceExpectations } = require('../fixtures/helpers.js'); + +const { runHttpTraceAssertions } = require('./http-trace-assertions.js'); + +let reportFilePath; +let tracesFilePath; +beforeEach(async (t) => { + reportFilePath = generateTmpReportPath(t.name, 'json'); + tracesFilePath = generateTmpReportPath('spans_' + t.name, 'json'); +}); + +afterEach(async (t) => { + deleteFile(reportFilePath); + deleteFile(tracesFilePath); +}); + +/* To write a test for the publish-metrics http tracing you need to: + 1. Define the test configuration through the override object + 2. Define the expected outcome values in the expectedOutcome object (see the required properties in the runHttptTraceAssertions function in the http-trace-assertions.js file) + 3. Run the test + 5. Assemble all test run data into one object for assertions (output of the test run, artillery report summary and exported spans) + 6. Run assertions with `runHttpTraceAssertions` + + NOTE: Any changes or features that influence the trace format or require additional checks + should be added to the `runHttpTraceAssertions` function +*/ + +test('OTel reporter correctly records trace data for http engine test runs', async (t) => { + // Define test configuration + const override = { + config: { + plugins: { + 'publish-metrics': [ + { + type: 'open-telemetry', + traces: { + exporter: '__test', + __outputPath: tracesFilePath, + useRequestNames: true, + replaceSpanNameRegex: [{ pattern: 'armadillo', as: 'bombolini' }], + attributes: { + environment: 'test', + tool: 'Artillery' + } + } + } + ] + } + } + }; + + // Define the expected outcome + const expectedOutcome = { + scenarioName: 'trace-http-test', + exitCode: 0, + vus: 4, + reqPerVu: 3, + reqSpansPerVu: 3, + vusFailed: 0, + errors: 0, + spansWithErrorStatus: 0, + userSetAttributes: + override.config.plugins['publish-metrics'][0].traces.attributes, + spanNamesByReqName: ['dino', 'bombolini'], + spanNamesByMethod: ['get'], + spanNamesReplaced: ['bombolini'] + }; + + // Setting expected values calculated from the base values + setDynamicHTTPTraceExpectations(expectedOutcome); + + /// Run the test + let output; + try { + output = + await $`artillery run ${__dirname}/../fixtures/http-trace.yml -o ${reportFilePath} --overrides ${JSON.stringify( + override + )}`; + } catch (err) { + console.error('There has been an error in test run execution: ', err); + t.fail(err); + } + + // Get all main test run data + const testRunData = { + output, + reportSummary: JSON.parse(fs.readFileSync(reportFilePath, 'utf8')) + .aggregate, + spans: JSON.parse(fs.readFileSync(tracesFilePath, 'utf8')) + }; + + // Run assertions + try { + await runHttpTraceAssertions(t, testRunData, expectedOutcome); + } catch (err) { + console.error(err); + } +}); + +test('OTel reporter works appropriately with "parallel" scenario setting ', async (t) => { + // Define test configuration + const override = { + config: { + plugins: { + 'publish-metrics': [ + { + type: 'open-telemetry', + traces: { + exporter: '__test', + __outputPath: tracesFilePath, + useRequestNames: true, + replaceSpanNameRegex: [{ pattern: 'armadillo', as: 'bombolini' }], + attributes: { + environment: 'test', + tool: 'Artillery' + } + } + } + ] + } + }, + scenarios: [ + { + name: 'trace-http-test', + flow: [ + { + parallel: [ + { get: { url: '/dino', name: 'dino' } }, + { get: { url: '/pony' } }, + { get: { url: '/armadillo', name: 'armadillo' } } + ] + } + ] + } + ] + }; + + // Define the expected outcome + const expectedOutcome = { + scenarioName: 'trace-http-test', + exitCode: 0, + vus: 4, + reqPerVu: 3, + reqSpansPerVu: 3, + vusFailed: 0, + errors: 0, + spansWithErrorStatus: 0, + userSetAttributes: + override.config.plugins['publish-metrics'][0].traces.attributes, + spanNamesByReqName: ['dino', 'bombolini'], + spanNamesByMethod: ['get'], + spanNamesReplaced: ['bombolini'] + }; + + // Setting expected values calculated from the base values + setDynamicHTTPTraceExpectations(expectedOutcome); + + /// Run the test + let output; + try { + output = + await $`artillery run ${__dirname}/../fixtures/http-trace.yml -o ${reportFilePath} --overrides ${JSON.stringify( + override + )}`; + } catch (err) { + t.fail(err); + } + + // Get all main test run data + const testRunData = { + output, + reportSummary: JSON.parse(fs.readFileSync(reportFilePath, 'utf8')) + .aggregate, + spans: JSON.parse(fs.readFileSync(tracesFilePath, 'utf8')) + }; + + // Run assertions + try { + await runHttpTraceAssertions(t, testRunData, expectedOutcome); + } catch (err) { + console.error(err); + } +}); + +test('Otel reporter appropriately records traces for test runs with errors', async (t) => { + // Define test configuration + const override = { + config: { + plugins: { + 'publish-metrics': [ + { + type: 'open-telemetry', + traces: { + exporter: '__test', + __outputPath: tracesFilePath, + useRequestNames: true, + replaceSpanNameRegex: [{ pattern: 'armadillo', as: 'bombolini' }], + attributes: { + environment: 'test', + tool: 'Artillery' + } + } + } + ] + } + }, + scenarios: [ + { + name: 'trace-http-test', + flow: [ + { + parallel: [ + { get: { url: '/dino', name: 'dino' } }, + { get: { url: '/armadillo', name: 'armadillo' } }, + { get: { url: '/pony', body: { json: 'This will fail' } } } + ] + } + ] + } + ] + }; + + // Define the expected outcome + const expectedOutcome = { + scenarioName: 'trace-http-test', + exitCode: 0, + vus: 4, + reqPerVu: 2, + reqSpansPerVu: 3, + reqSpansWithErrorPerVu: 1, + vusFailed: 4, + errors: 4, + spansWithErrorStatus: 4, + userSetAttributes: + override.config.plugins['publish-metrics'][0].traces.attributes, + spanNamesByReqName: ['dino', 'bombolini'], + spanNamesByMethod: ['get'], + spanNamesReplaced: ['bombolini'] + }; + + // Setting expected values calculated from the base values + setDynamicHTTPTraceExpectations(expectedOutcome); + + /// Run the test + let output; + try { + output = + await $`artillery run ${__dirname}/../fixtures/http-trace.yml -o ${reportFilePath} --overrides ${JSON.stringify( + override + )}`; + } catch (err) { + t.fail(err); + } + + // Get all main test run data + const testRunData = { + output, + reportSummary: JSON.parse(fs.readFileSync(reportFilePath, 'utf8')) + .aggregate, + spans: JSON.parse(fs.readFileSync(tracesFilePath, 'utf8')) + }; + + // Run assertions + try { + await runHttpTraceAssertions(t, testRunData, expectedOutcome); + } catch (err) { + console.error(err); + } +}); diff --git a/packages/artillery/test/publish-metrics/tracing/playwright-trace-assertions.js b/packages/artillery/test/publish-metrics/tracing/playwright-trace-assertions.js new file mode 100644 index 0000000000..17075fe7f1 --- /dev/null +++ b/packages/artillery/test/publish-metrics/tracing/playwright-trace-assertions.js @@ -0,0 +1,301 @@ +'use strict'; + +const { getTestId } = require('../fixtures/helpers.js'); + +/** Runs assertions for OTel Playwright tracing tests using 'tap' library. It checks that the trace data is correctly recorded, formatted and exported by the OTel plugin. + * @param {Object} t - the tap library test object + * @param {Object} testRunData - an object containing the console output of the test run, the report summary and the exported spans - `{ output, reportSummary, spans }` + * @namespace expectedOutcome + * @param {Object} expectedOutcome - an object containing the expected outcome values for the test run. + * @param {string} expectedOutcome.scenarioName - the name of the scenario + * @param {number} expectedOutcome.exitCode - the expected exit code of the test run + * @param {number} expectedOutcome.vus - the number of VUs created + * @param {number} [expectedOutcome.vusFailed] - the number of VUs that should fail + * @param {number} [expectedOutcome.errors] - the number of errors to be reported + * @param {number} [expectedOutcome.spansWithErrorStatus] - the number of spans that should have the error status + * @param {number} expectedOutcome.pageSpansPerVu - the number of page spans per VU + * @param {Array} expectedOutcome.pageSpanNames - an array of page span names expected - the final version of names when `replaceSpanRegex` is used + * @param {Array} expectedOutcome.pagesVisitedPerVU - an array of page URLs to be visited by each VU + * @param {number} [expectedOutcome.stepSpansPerVu] - the number of step spans per VU + * @param {Array} [expectedOutcome.stepNames] - an array of step names defined in the testFunction - the final version of names when `replaceSpanRegex` is used + * @param {Object} [expectedOutcome.userSetAttributes] - an object containing the user set attributes + * @param {Object} [expectedOutcome.modifiedSpanNames] - an object containing the expected modified span names + * @param {Array} [expectedOutcome.modifiedSpanNames.steps] - an array of step names expected - the final version of names when `replaceSpanRegex` is used + * @param {Array} [expectedOutcome.modifiedSpanNames.pages] - an array of page span names expected - the final version of names when `replaceSpanRegex` is used + * @param {number} [expectedOutcome.spansPerVu] - the number of spans per VU - `setDynamicPlaywrightTraceExpectations` function can be used to calculate this value + * @param {number} expectedOutcome.pageSpans - the number of page spans to be created - `setDynamicPlaywrightTraceExpectations` function can be used to calculate this value + * @param {number} [expectedOutcome.stepSpans] - the number of step spans to be created - `setDynamicPlaywrightTraceExpectations` function can be used to calculate this value + * @param {number} expectedOutcome.totalSpans - the total number of spans to be created - `setDynamicPlaywrightTraceExpectations` function can be used to calculate this value + * + * + * #### Configuration settings and functionality covered: + * - `replaceSpanNameRegex` config setting - replaces the specified pattern in page span names + * - `attributes` config setting - sets the user attributes for all spans + * - default attributes - `test_id` and `vu.uuid` attributes for all spans + * - web vitals recording - adds the web vitals values and ratings as attributes and events to the page spans when reported + * - navigation events - adds the navigation events to the scenario spans + * - `plugins.publish-metrics.spans.exported` metric - emits the counter metric for the number of spans exported + * - errors - errors are recorded on traces both as error events and as the error status code + * + * + * #### Configuration settings and functionality not covered - needs to be implemented in the future: + * - `sampleRate` config setting - lose percentage of spans to be sampled + * + * If any new features are added that add to or change the tracing format, this is where the change should be implemented to propagate to all tests. + */ + +async function runPlaywrightTraceAssertions(t, testRunData, expectedOutcome) { + const { output, reportSummary, spans } = testRunData; + const testId = getTestId(output.stdout); + + const scenarioSpans = spans.filter((span) => !span.parentId); + const pageSpans = spans.filter( + (span) => + (span.name.startsWith('Page') || span.attributes.url) && span.parentId + ); + const stepSpans = spans.filter( + (span) => + (!span.name.startsWith('Page') || !span.attributes.url) && span.parentId + ); + const stepsReported = Object.keys(reportSummary.summaries) + .filter((metricName) => metricName.startsWith('browser.step.')) + .map((metricName) => metricName.replace('browser.step.', '')); + + // Span counts + t.equal( + reportSummary.counters['vusers.created'], + expectedOutcome.vus, + `${expectedOutcome.vus} VUs should have been created` + ); + t.equal( + [...new Set(spans.map((span) => span.traceId))].length, + reportSummary.counters['vusers.created'], + 'The number of traces should match the number of VUs created' + ); + t.equal( + scenarioSpans.length, + reportSummary.counters['vusers.created'], + 'The number of scenario spans should match the number of VUs created' + ); + t.equal( + pageSpans.length, + expectedOutcome.pageSpans, + `${expectedOutcome.pageSpans} page spans should have been created` + ); + + expectedOutcome.stepNames.forEach((name) => { + t.ok( + stepsReported.includes(name), + 'All expected steps should have been reported' + ); + }); + t.equal( + stepSpans.length, + expectedOutcome.stepSpans, + `${expectedOutcome.stepSpans} step spans should have been created` + ); + t.equal( + spans.length, + expectedOutcome.totalSpans, + `There should be ${expectedOutcome.totalSpans} spans created in total` + ); + + // Counter metric reported + t.equal( + reportSummary.counters['plugins.publish-metrics.spans.exported'], + expectedOutcome.totalSpans, + 'The `plugins.publish-metrics.spans.exported` counter should match the total number of spans exported' + ); + + // Errors and failed VUs + const errorsReported = Object.keys(reportSummary.counters).filter( + (metricName) => metricName.startsWith('errors.') + ); + const numErrorsReported = errorsReported.reduce( + (acc, metricName) => acc + reportSummary.counters[metricName], + 0 + ); + const spansWithErrorStatus = spans.filter((span) => span.status.code === 2); + const spansWithErrorEvents = spans.filter((span) => + span.events.some((event) => event.name === 'exception') + ); + + t.equal( + output.exitCode, + expectedOutcome.exitCode, + `CLI Exit Code should be ${expectedOutcome.exitCode}` + ); + t.equal( + reportSummary.counters['vusers.failed'], + expectedOutcome.vusFailed, + `${expectedOutcome.vusFailed} VUs should have failed` + ); + t.equal( + numErrorsReported, + expectedOutcome.errors, + `There should be ${expectedOutcome.errors} errors reported` + ); + + // Span status can be set to error even when no error is recorded so we check status separately from error events + t.equal( + spansWithErrorStatus.length, + expectedOutcome.spansWithErrorStatus, + `${expectedOutcome.spansWithErrorStatus} spans should have the error status` + ); + + t.equal( + spansWithErrorEvents.length, + numErrorsReported, + 'Num of errors in report should match the num of spans with the error status' + ); + t.ok( + spansWithErrorEvents.every((span) => span.status.code === 2), + 'The error status code should be set on all spans with error events' + ); + + // `replaceSpanNameRegex` should replace the specified pattern in page span names + if (expectedOutcome.modifiedSpanNames) { + const numStepSpansPerStep = + stepSpans.length / expectedOutcome.stepNames.length; + expectedOutcome.modifiedSpanNames.steps.forEach((stepName) => { + t.equal( + stepSpans.filter((span) => span.name === stepName).length, + numStepSpansPerStep, + `All step spans should have the modified name '${stepName}'` + ); + }); + + const numPageSpansPerPage = + pageSpans.length / expectedOutcome.pageSpanNames.length; + expectedOutcome.modifiedSpanNames.pages.forEach((pageName) => { + t.equal( + pageSpans.filter((span) => span.name === pageName).length, + numPageSpansPerPage, + `All page spans should have the modified name '${pageName}'` + ); + }); + } + // Per VU/trace (this will check that each trace is nested correctly and all its data is where and what it is supposed to be): + scenarioSpans.forEach((span) => { + t.equal( + span.name, + expectedOutcome.scenarioName, + 'The root span should be named after the scenario' + ); + // each scenario span should have expected num of page spans and step spans + const pages = pageSpans + .filter((pageSpan) => pageSpan.parentId === span.id) + .map((pageSpan) => pageSpan.name); + const steps = stepSpans + .filter((stepSpan) => stepSpan.parentId === span.id) + .map((stepSpan) => stepSpan.name); + const eventNames = span.events.map((event) => event.name); + + t.equal( + pages.length, + expectedOutcome.pageSpansPerVu, + `Each scenario span should have ${expectedOutcome.pageSpansPerVu} page spans` + ); + t.equal( + steps.length, + expectedOutcome.stepSpansPerVu, + `Each scenario span should have ${expectedOutcome.stepSpansPerVu} step spans` + ); + + // each scenario span has the appropriate page and step spans - by name + expectedOutcome.stepNames.forEach((name) => { + t.ok( + steps.includes(name), + `Each scenario span should have a step span named '${name}'` + ); + }); + expectedOutcome.pageSpanNames.forEach((name) => { + t.ok( + pages.includes(name), + `Each scenario span should have a page span named '${name}'` + ); + }); + + // each scenario span has the appropriate navigation events + const navigationEvents = span.events + .map((event) => event.name) + .filter((eventName) => eventName.startsWith('navigated to')); + t.equal( + navigationEvents.length, + expectedOutcome.pageSpansPerVu, + 'The number of navigation events should match the number of pages visited' + ); + + expectedOutcome.pagesVisitedPerVU.forEach((page) => { + t.ok( + eventNames.includes(`navigated to ${page}`), + `Each scenario span should have a navigation event for '${page}'` + ); + }); + }); + + // Attributes + spans.forEach((span) => { + t.has( + span.attributes, + expectedOutcome.userSetAttributes, + 'All spans should have the user set attributes' + ); + t.hasProps( + span.attributes, + ['test_id', 'vu.uuid'], + 'All spans should have the test_id and vu.uuid attributes' + ); + t.equal( + span.attributes['test_id'], + testId, + 'All spans should have the correct test_id attribute value' + ); + }); + + // Web Vitals + const webVitals = ['LCP', 'FCP', 'CLS', 'TTFB', 'INP', 'FID']; + + // Since the web vitals are not reported consistently for all pages or vusers: + // - we get all web vitals reported for the test run + const webVitalMetricsReported = Object.keys(reportSummary.summaries).filter( + (metricName) => + metricName.startsWith('browser.page') && + webVitals.includes(metricName.split('.')[2]) + ); + + // - group the page spans by url + const pageSpansPerUrl = pageSpans.reduce((acc, pageSpan) => { + // console.log('PAGE SPAN URL: ', pageSpan.attributes.url) + if (!acc[pageSpan.attributes.url]) { + acc[pageSpan.attributes.url] = []; + } + acc[pageSpan.attributes.url].push(pageSpan); + return acc; + }, {}); + + // - check that the web vitals reported for a page are added to some of its page spans as attributes and events (vitals are aggregated by url in report so we can not check for exact match of vitals for each page span) + webVitalMetricsReported.forEach((metricName) => { + // the metric name format is 'browser.page.[vital].[url]' + const [, , vital, ...urlArr] = metricName.split('.'); + const url = urlArr.join('.'); + t.ok( + pageSpansPerUrl[url].some( + (pageSpan) => + pageSpan.attributes.hasOwnProperty(`web_vitals.${vital}.value`) && + pageSpan.attributes.hasOwnProperty(`web_vitals.${vital}.rating`) + ), + `${vital} value and rating reported for '${url}' should be added to its page span` + ); + t.ok( + pageSpansPerUrl[url].some((pageSpan) => + pageSpan.events.some((event) => event.name === vital) + ), + `${vital} web vital reported for '${url}' should be added to its page span as an event` + ); + }); +} + +module.exports = { + runPlaywrightTraceAssertions +}; diff --git a/packages/artillery/test/publish-metrics/tracing/playwright-trace.test.js b/packages/artillery/test/publish-metrics/tracing/playwright-trace.test.js new file mode 100644 index 0000000000..4264d4778f --- /dev/null +++ b/packages/artillery/test/publish-metrics/tracing/playwright-trace.test.js @@ -0,0 +1,212 @@ +const { test, afterEach, beforeEach } = require('tap'); +const { $ } = require('zx'); +const fs = require('fs'); +const { generateTmpReportPath, deleteFile } = require('../../cli/_helpers.js'); + +const { + setDynamicPlaywrightTraceExpectations +} = require('../fixtures/helpers.js'); + +const { + runPlaywrightTraceAssertions +} = require('./playwright-trace-assertions.js'); + +let reportFilePath; +let tracesFilePath; +beforeEach(async (t) => { + reportFilePath = generateTmpReportPath(t.name, 'json'); + tracesFilePath = generateTmpReportPath('spans_' + t.name, 'json'); +}); + +afterEach(async (t) => { + deleteFile(reportFilePath); + deleteFile(tracesFilePath); +}); + +/* To write a test for the publish-metrics tracing you need to: + 1. Define the test configuration through the override object + 2. Define the expected outcome values in the expectedOutcome object (see the required properties in the runPlaywrightTraceAssertions function in the playwright-trace-assertions.js file) + 3. Run the test + 5. Assemble all test run data into one object for assertions (output of the test run, artillery report summary and exported spans) + 6. Run assertions with `runPlaywrightTraceAssertions` + + NOTE: Any changes or features that influence the trace format or require additional checks + should be added to the `runPlaywrightTraceAssertions` function +*/ + +test('OTel reporter correctly records trace data for playwright engine test runs', async (t) => { + // Define test configuration + const override = { + config: { + plugins: { + 'publish-metrics': [ + { + type: 'open-telemetry', + traces: { + exporter: '__test', + __outputPath: tracesFilePath, + replaceSpanNameRegex: [ + { + pattern: + 'https://www.artillery.io/docs/get-started/core-concepts', + as: 'core_concepts' + }, + { pattern: 'https://www.artillery.io/docs', as: 'docs_main' }, + { pattern: 'Go to core concepts', as: 'bombolini' } + ], + attributes: { + environment: 'test', + tool: 'Artillery' + } + } + } + ] + } + } + }; + + // Define the expected outcome + const expectedOutcome = { + scenarioName: 'trace-playwright-test', + exitCode: 0, + vus: 4, + vusFailed: 0, + errors: 0, + spansWithErrorStatus: 0, + pageSpansPerVu: 3, + stepSpansPerVu: 3, + userSetAttributes: + override.config.plugins['publish-metrics'][0].traces.attributes, + stepNames: ['Go to Artillery', 'Go to docs', 'bombolini'], + pageSpanNames: [ + 'Page: https://www.artillery.io/', + 'Page: docs_main', + 'Page: core_concepts' + ], + pagesVisitedPerVU: [ + 'https://www.artillery.io/', + 'https://www.artillery.io/docs', + 'https://www.artillery.io/docs/get-started/core-concepts' + ], + modifiedSpanNames: { + steps: ['bombolini'], + pages: ['Page: docs_main', 'Page: core_concepts'] + } + }; + + setDynamicPlaywrightTraceExpectations(expectedOutcome); + + // Run the test + let output; + try { + output = + await $`artillery run ${__dirname}/../fixtures/playwright-trace.yml -o ${reportFilePath} --overrides ${JSON.stringify( + override + )}`; + } catch (err) { + t.fail(err); + } + + // Assemble all test run data into one object for assertions (output of the test run, artillery report summary and exported spans) + const testRunData = { + output, + reportSummary: JSON.parse(fs.readFileSync(reportFilePath, 'utf8')) + .aggregate, + spans: JSON.parse(fs.readFileSync(tracesFilePath, 'utf8')) + }; + + // Run assertions + try { + await runPlaywrightTraceAssertions(t, testRunData, expectedOutcome); + } catch (err) { + console.error(err); + t.fail(err); + } +}); + +test('OTel reporter correctly records trace data for playwright engine test runs', async (t) => { + // Define test configuration + const override = { + config: { + plugins: { + 'publish-metrics': [ + { + type: 'open-telemetry', + traces: { + exporter: '__test', + __outputPath: tracesFilePath, + replaceSpanNameRegex: [ + { pattern: 'https://www.artillery.io/docs', as: 'docs_main' }, + { pattern: 'Go to core concepts', as: 'bombolini' } + ], + attributes: { + environment: 'test', + tool: 'Artillery' + } + } + } + ] + } + }, + scenarios: [ + { + name: 'trace-playwright-test', + engine: 'playwright', + testFunction: 'simpleError' + } + ] + }; + + // Define the expected outcome + const expectedOutcome = { + scenarioName: 'trace-playwright-test', + exitCode: 0, + vus: 4, + vusFailed: 4, + errors: 4, + spansWithErrorStatus: 4, + pageSpansPerVu: 2, + stepSpansPerVu: 3, + userSetAttributes: + override.config.plugins['publish-metrics'][0].traces.attributes, + stepNames: ['Go to Artillery', 'Go to docs', 'bombolini'], + pageSpanNames: ['Page: https://www.artillery.io/', 'Page: docs_main'], + pagesVisitedPerVU: [ + 'https://www.artillery.io/', + 'https://www.artillery.io/docs' + ], + modifiedSpanNames: { + steps: ['bombolini'], + pages: ['Page: docs_main'] + } + }; + + setDynamicPlaywrightTraceExpectations(expectedOutcome); + + // Run the test + let output; + try { + output = + await $`artillery run ${__dirname}/../fixtures/playwright-trace.yml -o ${reportFilePath} --overrides ${JSON.stringify( + override + )}`; + } catch (err) { + t.fail(err); + } + + // Assembling all test run data into one object for assertions (output of the test run, artillery report summary and exported spans) + const testRunData = { + output, + reportSummary: JSON.parse(fs.readFileSync(reportFilePath, 'utf8')) + .aggregate, + spans: JSON.parse(fs.readFileSync(tracesFilePath, 'utf8')) + }; + + // Run assertions + try { + await runPlaywrightTraceAssertions(t, testRunData, expectedOutcome); + } catch (err) { + console.error(err); + t.fail(err); + } +});