From fba2acfdb48fb1080a4a900f615ebcae00aff9d8 Mon Sep 17 00:00:00 2001 From: GALLLASMILAN Date: Fri, 20 Dec 2024 10:49:14 +0100 Subject: [PATCH] feat(docker): integration tests in github pipeline Signed-off-by: GALLLASMILAN --- .env.example | 2 +- .env.testing.docker | 5 +++- .github/workflows/test.yml | 51 +++++++++++++++++++++++++++++++++++ compose-before.yml | 38 -------------------------- compose.yml | 42 +++++++++++++++++++++++++++++ package.json | 4 +-- scripts/test_local.sh | 18 ++----------- src/span/span.test.ts | 9 ++----- src/testing/utils.ts | 55 ++++++++++++++++++++++++++++++++++++++ src/trace/trace.test.ts | 6 ++--- 10 files changed, 162 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 compose-before.yml diff --git a/.env.example b/.env.example index 8051c41..f57bddd 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ ################# Fastify ################# ########################################### # 1. Fastify port -PORT=3000 +PORT=4318 # 2. Secure auth API key AUTH_KEY=very-strong-and-long-api-key-with-special-chars # 3. Request body limit diff --git a/.env.testing.docker b/.env.testing.docker index 0b5d42e..0ed567b 100644 --- a/.env.testing.docker +++ b/.env.testing.docker @@ -1,3 +1,4 @@ +NODE_ENV=production PORT=4318 AUTH_KEY=valid-api-key FASTIFY_BODY_LIMIT=10485760 @@ -11,8 +12,10 @@ MLFLOW_AUTHORIZATION=BASE_AUTH MLFLOW_USERNAME=admin MLFLOW_PASSWORD=password MLFLOW_DEFAULT_EXPERIMENT_ID=0 -MLFLOW_TRACE_DELETE_IN_BATCHES_CRON_PATTERN="0 */1 * * * *" +MLFLOW_TRACE_DELETE_IN_BATCHES_CRON_PATTERN="* * * * */20" MLFLOW_TRACE_DELETE_IN_BATCHES_BATCH_SIZE=100 BEE_FRAMEWORK_INSTRUMENTATION_METRICS_ENABLED=false BEE_FRAMEWORK_INSTRUMENTATION_ENABLED=true + +USE_FAKE_BACKEND_FOR_TESTING=true \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e02e011 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,51 @@ +name: Integration Test + +on: + push: + branches: ['main'] + pull_request: + branches: ['main'] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + main: + timeout-minutes: 20 + name: Integration Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Enable Corepack + run: corepack enable + - name: Use Node.js + uses: actions/setup-node@v4 + with: + node-version-file: '.nvmrc' + cache: 'yarn' + - name: Install dependencies + run: yarn install + - name: Copy .env file + run: cp .env.testing.docker .env + - name: Build infra + run: docker compose build + - name: Build Observe API + run: docker compose build + - name: Temp 1 + run: docker ps + - name: Run Observe API + run: docker compose up -d + - name: Temp 2 + run: docker ps + - name: Run Vitest + run: yarn run vitest --run + - name: Collect Logs on Failure (Observe) + if: failure() + run: docker compose logs + - name: Cleanup + if: always() + run: docker compose down diff --git a/compose-before.yml b/compose-before.yml deleted file mode 100644 index 251605c..0000000 --- a/compose-before.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: 'bee-observe' -services: - mongo: - image: mongo:7 - ports: - - '${DATABASE_EXPOSED_PORT:-27019}:27017' - environment: - MONGO_INITDB_ROOT_USERNAME: ${MONGO_INITDB_ROOT_USERNAME:-mongo} - MONGO_INITDB_ROOT_PASSWORD: ${MONGO_INITDB_ROOT_PASSWORD:-mongo} - healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet - interval: 10s - timeout: 5s - retries: 5 - redis: - image: redis:7 - ports: - - '${REDIS_EXPOSED_PORT:-6380}:6379' - command: redis-server --save 20 1 --loglevel warning - healthcheck: - test: ['CMD-SHELL', 'redis-cli ping | grep PONG'] - interval: 10s - timeout: 5s - retries: 5 - mlflow: - image: bitnami/mlflow:2.17.2 - ports: - - '${MLFLOW_EXPOSED_PORT:-8080}:8080' - entrypoint: - [ - '/bin/bash', - '-c', - '/entrypoint.sh && mlflow server --app-name basic-auth --host 0.0.0.0 --port 8080' - ] - security_opt: - - 'label=disable' - volumes: - - ./entrypoint.sh:/entrypoint.sh:ro diff --git a/compose.yml b/compose.yml index afe8d59..aff740b 100644 --- a/compose.yml +++ b/compose.yml @@ -1,5 +1,41 @@ name: 'bee-observe' services: + mongo: + image: mongo:7 + ports: + - '${DATABASE_EXPOSED_PORT:-27019}:27017' + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGO_INITDB_ROOT_USERNAME:-mongo} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_INITDB_ROOT_PASSWORD:-mongo} + healthcheck: + test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017/test --quiet + interval: 10s + timeout: 5s + retries: 5 + redis: + image: redis:7 + ports: + - '${REDIS_EXPOSED_PORT:-6380}:6379' + command: redis-server --save 20 1 --loglevel warning + healthcheck: + test: ['CMD-SHELL', 'redis-cli ping | grep PONG'] + interval: 10s + timeout: 5s + retries: 5 + mlflow: + image: bitnami/mlflow:2.17.2 + ports: + - '${MLFLOW_EXPOSED_PORT:-8080}:8080' + entrypoint: + [ + '/bin/bash', + '-c', + '/entrypoint.sh && mlflow server --app-name basic-auth --host 0.0.0.0 --port 8080' + ] + security_opt: + - 'label=disable' + volumes: + - ./entrypoint.sh:/entrypoint.sh:ro observe_api: build: context: . @@ -10,6 +46,12 @@ services: - '${OBSERVE_API_EXPOSED_PORT:-4318}:4318' env_file: - .env.testing.docker + command: > + /bin/sh -c " + touch tsconfig.json && + NODE_ENV=production npx mikro-orm --config dist/mikro-orm.config.js migration:up && + node ./dist/index.js + " healthcheck: test: wget --no-verbose --tries=1 --spider http://0.0.0.0:4318/health || exit 1 interval: 10s diff --git a/package.json b/package.json index 29274a9..86e21ff 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,9 @@ "scripts": { "build": "rm -rf dist && tsc && cp -R src/protos dist/protos", "proto:generate": "./scripts/open_telemetry_generate_protos/generate_protos.sh", - "start:infra": "docker compose -f compose-before.yml up -d mongo redis mlflow", + "start:infra": "docker compose up -d mongo redis mlflow", "start:dev": "tsx watch ./src/index.ts | pino-pretty --singleLine", - "stop:infra": "docker compose -f compose-before.yml down", + "stop:infra": "docker compose down", "dev": "yarn start:dev", "start": "node ./dist/index.js", "test": "./scripts/test_local.sh", diff --git a/scripts/test_local.sh b/scripts/test_local.sh index 9f75fb2..c3654a5 100755 --- a/scripts/test_local.sh +++ b/scripts/test_local.sh @@ -19,7 +19,6 @@ set -e ## turn off testing containers if are running docker compose down -docker compose -f compose-before.yml down ## run containers ### pull latest versions @@ -28,18 +27,6 @@ docker compose pull ### build the observe image docker compose build -### run compose up and wait 120 seconds - in case of failure - cancel the operation and print logs -if timeout 120 yarn start:infra 2>/dev/null ; then - echo '🆗 docker containers are working and ready to test' -else - echo '❌ There is some error with the docker compose up command. Check your environments in .env.testing.docker and see docker logs ...' - docker compose logs - exit 1; -fi - -### run migrations -yarn mikro-orm-esm migration:up - ### run compose up and wait 120 seconds - in case of failure - cancel the operation and print logs if timeout 120 docker compose up -d 2>/dev/null ; then echo '🆗 docker containers are working and ready to test' @@ -52,9 +39,8 @@ fi ## run integration tests yarn run vitest --run -## 7) clean +## clean docker compose down -docker compose -f compose-before.yml down -## 8) info +## info echo "✅ tests done" diff --git a/src/span/span.test.ts b/src/span/span.test.ts index 940e145..b4033e3 100644 --- a/src/span/span.test.ts +++ b/src/span/span.test.ts @@ -18,7 +18,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { Version } from 'bee-agent-framework'; import { sdk, spanTraceExporterProcessor } from '../testing/telemetry.js'; -import { agent, makeRequest } from '../testing/utils.js'; +import { generateTrace, makeRequest } from '../testing/utils.js'; let traceId: string | undefined = undefined; const prompt = 'hello'; @@ -26,7 +26,7 @@ const prompt = 'hello'; describe('span module', () => { beforeAll(async () => { await sdk.start(); - await agent.run({ prompt }).middleware((ctx) => (traceId = ctx.emitter.trace?.id)); + traceId = await generateTrace({ prompt }); await spanTraceExporterProcessor.forceFlush(); }); @@ -63,10 +63,5 @@ describe('span module', () => { expect(mainSpan.attributes.prompt).toBe(prompt); expect(mainSpan.attributes.response.text.length).toBeGreaterThan(0); expect(mainSpan.ctx).toBeUndefined(); - - const startSpan = results.find((result: any) => result.name === 'ollama.chat_llm.start-1'); - expect( - startSpan.attributes.data.input.find((input: any) => input.role === 'system').text.length - ).toBeGreaterThan(0); }); }); diff --git a/src/testing/utils.ts b/src/testing/utils.ts index c7fffaa..ad36a2c 100644 --- a/src/testing/utils.ts +++ b/src/testing/utils.ts @@ -15,6 +15,7 @@ */ import { setTimeout as setTimeoutPromise } from 'node:timers/promises'; +import { api } from '@opentelemetry/sdk-node'; import { BeeAgent } from 'bee-agent-framework/agents/bee/agent'; import { TokenMemory } from 'bee-agent-framework/memory/tokenMemory'; @@ -26,6 +27,7 @@ import protobuf from 'protobufjs'; import { TraceDto } from '../trace/trace.dto.js'; import { constants } from '../utils/constants.js'; +import { Version } from 'bee-agent-framework'; export const buildUrl = (route: string): string => { const port = process.env.PORT || '4318'; @@ -131,3 +133,56 @@ export async function sendCustomProtobuf(payload: SendCustomProtobufProps) { body: buffer }); } + +const fakeSpans = [ + { + id: 'fake-1', + target: 'groupId', + name: 'iteration-1' + } +]; + +export async function generateTrace({ prompt }: { prompt: string; }) { + let traceId: string | undefined = undefined; + // bee-agent-framework + if (!process.env.USE_FAKE_BACKEND_FOR_TESTING) { + await agent.run({ prompt }).middleware((ctx) => (traceId = ctx.emitter.trace?.id)); + return traceId; + } + + // mock + const tracer = api.trace.getTracer("bee-agent-framework", Version); + traceId = 'ea215b9f'; + + // 1) main span + tracer.startActiveSpan(`bee-agent-framework-BeeAgent-${traceId}`, { + attributes: { + traceId, + version: Version, + prompt, + response: JSON.stringify({ + role: "assistant", + text: "Hello! It's nice to chat with you." + }) + } + }, (activeSpan) => { + activeSpan.setStatus({ code: 1 }); + + // 2) nested spans + fakeSpans.map((span) => { + tracer.startActiveSpan(span.id, { + attributes: { + target: span.target, + name: span.name + } + }, (nestedSpan) => { + nestedSpan.setStatus({ code: 1 }); + nestedSpan.end(); + }) + }); + + activeSpan.end(); + }); + + return traceId; +} diff --git a/src/trace/trace.test.ts b/src/trace/trace.test.ts index cfe7c2e..27b6f14 100644 --- a/src/trace/trace.test.ts +++ b/src/trace/trace.test.ts @@ -17,7 +17,7 @@ import { expect, it, describe, beforeAll, afterAll } from 'vitest'; import { sdk, spanTraceExporterProcessor } from '../testing/telemetry.js'; -import { agent, makeRequest, sendCustomProtobuf, waitForMlflowTrace } from '../testing/utils.js'; +import { generateTrace, makeRequest, sendCustomProtobuf, waitForMlflowTrace } from '../testing/utils.js'; let traceId: string | undefined = undefined; const prompt = 'hello'; @@ -25,7 +25,7 @@ const prompt = 'hello'; describe('trace module', () => { beforeAll(async () => { await sdk.start(); - await agent.run({ prompt }).middleware((ctx) => (traceId = ctx.emitter.trace?.id)); + traceId = await generateTrace({ prompt }); await spanTraceExporterProcessor.forceFlush(); if (traceId) await waitForMlflowTrace({ traceId }); }); @@ -41,7 +41,7 @@ describe('trace module', () => { it('should use "Retry-After" header and wait until the trace is ready', async () => { let retryAfterTraceId: string | undefined = undefined; - await agent.run({ prompt }).middleware((ctx) => (retryAfterTraceId = ctx.emitter.trace?.id)); + retryAfterTraceId = await generateTrace({ prompt }); if (retryAfterTraceId) await waitForMlflowTrace({ traceId: retryAfterTraceId }); const traceResponse = await makeRequest({ route: `v1/traces/${retryAfterTraceId}` });