improve web testing (#3162)

* shared admin level test dependency

* change to on - push (recommended by chromatic)

* change playwright reporter to list, name test jobs

* use test tags ... much cleaner

* test vs prod

* try copying templates

* run with localhost?

* revert to dev

* new tests and a bit of refactoring

* add additional checks so that page snapshots reflect loaded state

* more admin tests

* User Management tests

* remaining admin pages

* test search and chat

* await fix and exclude UI that changes with dates.

.github/workflows/pr-chromatic-tests.yml

@@ -3,19 +3,16 @@ concurrency:
   group: Run-Chromatic-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
   cancel-in-progress: true
 
-on:
-  merge_group:
-  pull_request:
-    branches:
-      - main
-      - 'release/**'
+on: push
 
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
 jobs:
   playwright-tests:
+    name: Playwright Tests
+
     # See https://runs-on.com/runners/linux/
     runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
     steps:
@@ -108,7 +105,7 @@ jobs:
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
           cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
-      - name: Start Docker containers 
+      - name: Start Docker containers
         run: |
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
@@ -193,7 +190,8 @@ jobs:
           docker compose -f docker-compose.dev.yml -p danswer-stack down -v
 
   chromatic-tests:
-    name: Run Chromatic
+    name: Chromatic Tests
+
     needs: playwright-tests
     runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
     steps:

deployment/docker_compose/README.md

@@ -5,7 +5,7 @@
 For general information, please read the instructions in this [README](https://github.com/danswer-ai/danswer/blob/main/deployment/README.md).
 
 ## Deploy in a system without GPU support
-This part is elaborated precisely in  in this [README](https://github.com/danswer-ai/danswer/blob/main/deployment/README.md) in section *Docker Compose*. If you have any questions, please feel free to open an issue or get in touch in slack for support.
+This part is elaborated precisely in this [README](https://github.com/danswer-ai/danswer/blob/main/deployment/README.md) in section *Docker Compose*. If you have any questions, please feel free to open an issue or get in touch in slack for support.
 
 ## Deploy in a system with GPU support
 Running Model servers with GPU support while indexing and querying can result in significant improvements in performance. This is highly recommended if you have access to resources. Currently, Danswer offloads embedding model and tokenizers to the GPU VRAM and the size needed depends on chosen embedding model. For example, the embedding model `nomic-ai/nomic-embed-text-v1` takes up about 1GB of VRAM. That means running this model for inference and embedding pipeline would require roughly 2GB of VRAM.

web/.gitignore

@@ -34,3 +34,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+/admin_auth.json
+/build-archive.log
+

web/playwright.config.ts

@@ -1,8 +1,8 @@
-import { defineConfig } from "@playwright/test";
+import { defineConfig, devices } from "@playwright/test";
 
 export default defineConfig({
-  // Other Playwright config options
   testDir: "./tests/e2e", // Folder for test files
+  reporter: "list",
   // Configure paths for screenshots
   // expect: {
   //   toMatchSnapshot: {
@@ -11,4 +11,30 @@ export default defineConfig({
   // },
   // reporter: [["html", { outputFolder: "test-results/output/report" }]], // HTML report location
   // outputDir: "test-results/output/screenshots", // Set output folder for test artifacts
+  projects: [
+    {
+      // dependency for admin workflows
+      name: "admin_setup",
+      testMatch: /.*\admin_auth.setup\.ts/,
+    },
+    {
+      // tests admin workflows
+      name: "chromium-admin",
+      grep: /@admin/,
+      use: {
+        ...devices["Desktop Chrome"],
+        // Use prepared auth state.
+        storageState: "admin_auth.json",
+      },
+      dependencies: ["admin_setup"],
+    },
+    {
+      // tests logged out / guest workflows
+      name: "chromium-guest",
+      grep: /@guest/,
+      use: {
+        ...devices["Desktop Chrome"],
+      },
+    },
+  ],
 });

web/src/app/admin/api-key/page.tsx

@@ -29,9 +29,7 @@ import { deleteApiKey, regenerateApiKey } from "./lib";
 import { DanswerApiKeyForm } from "./DanswerApiKeyForm";
 import { APIKey } from "./types";
 
-const API_KEY_TEXT = `
-API Keys allow you to access Danswer APIs programmatically. Click the button below to generate a new API Key.
-`;
+const API_KEY_TEXT = `API Keys allow you to access Danswer APIs programmatically. Click the button below to generate a new API Key.`;
 
 function NewApiKeyModal({
   apiKey,

web/tests/e2e/admin_add_connector.spec.ts

@@ -0,0 +1,14 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Connectors - Add Connector",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/add-connector");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Add Connector");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(/^Storage/);
+  }
+);

web/tests/e2e/admin_api_key.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - User Management - API Keys",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/api-key");
+    await expect(page.locator("h1.text-3xl")).toHaveText("API Keys");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      /^API Keys allow you to access Danswer APIs programmatically/
+    );
+    await expect(
+      page.getByRole("button", { name: "Create API Key" })
+    ).toHaveCount(1);
+  }
+);

web/tests/e2e/admin_assistants.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Custom Assistants - Assistants",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/assistants");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Assistants");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText(
+      /^Assistants are a way to build/
+    );
+  }
+);

web/tests/e2e/admin_auth.setup.ts

@@ -0,0 +1,24 @@
+// dependency for all admin user tests
+
+import { test as setup, expect } from "@playwright/test";
+import { TEST_CREDENTIALS } from "./constants";
+
+setup("authenticate", async ({ page }) => {
+  const { email, password } = TEST_CREDENTIALS;
+
+  await page.goto("http://localhost:3000/search");
+
+  await page.waitForURL("http://localhost:3000/auth/login?next=%2Fsearch");
+
+  await expect(page).toHaveTitle("Danswer");
+
+  await page.fill("#email", email);
+  await page.fill("#password", password);
+
+  // Click the login button
+  await page.click('button[type="submit"]');
+
+  await page.waitForURL("http://localhost:3000/search");
+
+  await page.context().storageState({ path: "admin_auth.json" });
+});

web/tests/e2e/admin_bots.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Custom Assistants - Slack Bots",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/bots");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Slack Bots");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText(
+      /^Setup Slack bots that connect to Danswer./
+    );
+  }
+);

web/tests/e2e/admin_configuration_document_processing.spec.ts

@@ -0,0 +1,18 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Configuration - Document Processing",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto(
+      "http://localhost:3000/admin/configuration/document-processing"
+    );
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Processing");
+    await expect(page.locator("h3.text-2xl")).toHaveText(
+      "Process with Unstructured API"
+    );
+  }
+);

web/tests/e2e/admin_configuration_llm.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Configuration - LLM",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/configuration/llm");
+    await expect(page.locator("h1.text-3xl")).toHaveText("LLM Setup");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(
+      "Enabled LLM Providers"
+    );
+  }
+);

web/tests/e2e/admin_configuration_search.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Configuration - Search Settings",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/configuration/search");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Search Settings");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(
+      "Embedding Model"
+    );
+  }
+);

web/tests/e2e/admin_documents_explorer.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Document Management - Feedback",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/documents/explorer");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Explorer");
+    await expect(page.locator("div.flex.text-emphasis.mt-3")).toHaveText(
+      "Search for a document above to modify its boost or hide it from searches."
+    );
+  }
+);

web/tests/e2e/admin_documents_feedback.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Document Management - Feedback",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/documents/feedback");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Feedback");
+    await expect(page.locator("h1.text-lg").nth(0)).toHaveText(
+      "Most Liked Documents"
+    );
+    await expect(page.locator("h1.text-lg").nth(1)).toHaveText(
+      "Most Disliked Documents"
+    );
+  }
+);

web/tests/e2e/admin_documents_sets.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Document Management - Document Sets",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/documents/sets");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Document Sets");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      /^Document Sets allow you to group logically connected documents into a single bundle./
+    );
+  }
+);

web/tests/e2e/admin_groups.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - User Management - Groups",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/groups");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Manage User Groups");
+    await expect(
+      page.getByRole("button", { name: "Create New User Group" })
+    ).toHaveCount(1);
+  }
+);

web/tests/e2e/admin_indexing_status.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Connectors - Existing Connectors",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/indexing/status");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Existing Connectors");
+    await expect(page.locator("p.text-sm")).toHaveText(
+      /^It looks like you don't have any connectors setup yet./
+    );
+  }
+);

web/tests/e2e/admin_performance_custom_analytics.spec.ts

@@ -0,0 +1,16 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Performance - Custom Analytics",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/performance/custom-analytics");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Custom Analytics");
+    await expect(page.locator("div.font-medium").nth(0)).toHaveText(
+      "Custom Analytics is not enabled."
+    );
+  }
+);

web/tests/e2e/admin_performance_query_history.spec.ts

@@ -0,0 +1,14 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test(
+  "Admin - Performance - Query History",
+  {
+    tag: "@admin",
+  },
+  async ({ page }, testInfo) => {
+    // Test simple loading
+    await page.goto("http://localhost:3000/admin/performance/query-history");
+    await expect(page.locator("h1.text-3xl")).toHaveText("Query History");
+    await expect(page.locator("p.text-sm").nth(0)).toHaveText("Feedback Type");
+  }
+);

web/tests/e2e/admin_performance_usage.spec.ts

@@ -0,0 +1,19 @@
+import { test, expect } from "@chromatic-com/playwright";
+
+test.describe("Admin Performance Usage", () => {
+  // Ignores the diff for elements targeted by the specified list of selectors
+  test.use({ ignoreSelectors: ["button", "svg"] });
+
+  test(
+    "Admin - Performance - Usage Statistics",
+    {
+      tag: "@admin",
+    },
+    async ({ page }, testInfo) => {
+      await page.goto("http://localhost:3000/admin/performance/usage");
+      await expect(page.locator("h1.text-3xl")).toHaveText("Usage Statistics");
+      await expect(page.locator("h1.text-lg").nth(0)).toHaveText("Usage");
+      await expect(page.locator("h1.text-lg").nth(1)).toHaveText("Feedback");
+    }
+  );
+});