C3: Better e2e quarantine dx (#4096)

* C3: Catch errors in quarantined e2e tests

* Address PR feedback

* fix lint issue

* Split up C3 workflow files and run quarantine e2es on pushed to main

* Pass accountId and token as inputs to e2e action

* Specify shell in github action

* Fix cleanup job trigger

* Allow quarantine tests to fail again

.github/actions/run-c3-e2e/action.yml

@@ -0,0 +1,42 @@
+name: "Run C3 E2E Tests"
+description: "Runs the C3 E2E tests on npm, pnpm, and bun"
+inputs:
+  package-manager:
+    description: "Which package manager to run the tests with"
+    required: true
+  quarantine:
+    description: "Whether to run the tests in quarantine mode"
+    required: false
+  framework:
+    description: "When specified, will only run tests for this framework"
+    required: false
+  accountId:
+    description: "The account id of the test account"
+    required: true
+  apiToken:
+    description: "The api token of the test account"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Bun ${{ env.bun-version }}
+      uses: oven-sh/setup-bun@v1
+      with:
+        bun-version: ${{ env.bun-version }}
+
+    # Needed because gatsby requires git
+    - name: Configure Git
+      shell: bash
+      run: |
+        git config --global user.email wrangler@cloudflare.com
+        git config --global user.name 'Wrangler automated PR updater'
+
+    - name: E2E Tests
+      shell: bash
+      run: pnpm run --filter create-cloudflare test:e2e:${{ inputs.package-manager }}
+      env:
+        CLOUDFLARE_API_TOKEN: ${{ inputs.apiToken }}
+        CLOUDFLARE_ACCOUNT_ID: ${{ inputs.accountId }}
+        E2E_QUARANTINE: ${{ inputs.quarantine }}
+        FRAMEWORK_CLI_TO_TEST: ${{ inputs.framework }}

.github/workflows/c3-checks.yml

@@ -0,0 +1,34 @@
+name: C3 Checks
+
+on:
+  pull_request:
+    paths:
+      - packages/create-cloudflare/**
+
+env:
+  node-version: 18.17.1
+
+jobs:
+  check:
+    name: "Checks"
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Dependencies
+        uses: ./.github/actions/install-dependencies
+
+      - name: Check Types
+        run: pnpm run --filter create-cloudflare check:type
+
+      - name: Lint
+        run: pnpm run --filter create-cloudflare check:lint
+
+      - name: Unit Tests
+        run: pnpm run --filter create-cloudflare test:unit

.github/workflows/c3-e2e-dependabot.yml

@@ -0,0 +1,86 @@
+# Runs the C3 e2e tests on dependabot PRs
+
+name: C3 E2E Tests (Dependabot)
+
+on:
+  pull_request:
+    paths:
+      - packages/create-cloudflare/**
+
+env:
+  node-version: 18.17.1
+  bun-version: 1.0.3
+
+jobs:
+  get-dependabot-bumped-framework:
+    name: "Get bumped framework (dependabot-only)"
+    runs-on: ubuntu-latest
+    outputs:
+      bumped-framework-cli: ${{ steps.detect.outputs.result }}
+    if: |
+      github.event.pull_request.user.login == 'dependabot[bot]'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          ref: ${{ github.head_ref }}
+      - name: Get PR description as string
+        id: get-pr-description
+        run: |
+          str=$(sed 's/`/\`/g' <<EOF
+            ${{ github.event.pull_request.body }}
+          EOF
+          )
+          echo 'result<<EOF' >> $GITHUB_OUTPUT
+          echo $str >> $GITHUB_OUTPUT
+          echo 'EOF' >> $GITHUB_OUTPUT
+      - name: detect-bumped-framework
+        id: detect
+        uses: actions/github-script@v6
+        with:
+          result-encoding: string
+          script: |
+            const json = require('./packages/create-cloudflare/src/frameworks/package.json')
+            const frameworkCliPackages = Object.values(json.frameworkCliMap);
+
+            const body = `${{ steps.get-pr-description.outputs.result }}`;
+
+            const semverRegexStr = '\\d+\\.\\d+\\.\\d+';
+
+            const frameworkCliRegex = new RegExp(
+              `(?:^|\\s+)Bumps\\s+\\[(${frameworkCliPackages.join(
+                '|'
+              )})\\]\\(.*?\\)\\s+from\\s+${semverRegexStr}\\s+to\\s+${semverRegexStr}`
+            );
+
+            const bumpedFrameworkCli = body.match(frameworkCliRegex)?.[1] ?? '';
+            return bumpedFrameworkCli;
+
+  # For dependabot versioning PRs we only want to run the e2es for the specifically bumped
+  # framework (this is both for optimization and in order to reduce unnecessary flakiness)
+  e2e-only-dependabot-bumped-framework:
+    needs: [get-dependabot-bumped-framework]
+    name: ${{ format('Dependabot E2E ({0})', matrix.pm) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        pm: [npm, pnpm, bun]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Dependencies
+        uses: ./.github/actions/install-dependencies
+
+      - name: E2E Tests
+        uses: ./.github/actions/run-c3-e2e
+        with:
+          package-manager: ${{ matrix.pm }}
+          framework: ${{ needs.get-dependabot-bumped-framework.outputs.bumped-framework-cli }}
+          accountId: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
+          apiToken: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}

.github/workflows/c3-e2e-project-cleanup.yml

@@ -0,0 +1,29 @@
+# This workflow cleans up any leftover projects created by e2e runs.
+
+name: C3 E2E Project Cleanup
+on:
+  workflow_run:
+    workflows: [C3 E2E Tests, C3 E2E Tests (Dependabot), C3 E2E (Quarantine)]
+    types:
+      - completed
+env:
+  node-version: 18.17.1
+jobs:
+  cleanup:
+    name: "Cleanup Test Projects"
+    if: ${{ github.repository_owner == 'cloudflare' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Dependencies
+        uses: ./.github/actions/install-dependencies
+
+      - name: Cleanup E2E test projects
+        run: pnpm run --filter create-cloudflare test:e2e:cleanup
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}

.github/workflows/c3-e2e-quarantine.yml

@@ -0,0 +1,36 @@
+# Runs c3 e2e tests for quarantined frameworks on merges to main
+
+name: C3 E2E (Quarantine)
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - packages/create-cloudflare/**
+env:
+  node-version: 18.17.1
+  bun-version: 1.0.3
+jobs:
+  e2e:
+    name: ${{ format('E2E ({0})', matrix.pm) }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        pm: [npm, pnpm, bun]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Dependencies
+        uses: ./.github/actions/install-dependencies
+
+      - name: E2E Tests
+        uses: ./.github/actions/run-c3-e2e
+        with:
+          package-manager: ${{ matrix.pm }}
+          quarantine: true
+          accountId: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
+          apiToken: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}

.github/workflows/c3-e2e.yml

@@ -0,0 +1,36 @@
+# Runs c3 e2e tests on pull requests with c3 changes
+
+name: C3 E2E Tests
+on:
+  pull_request:
+    paths:
+      - packages/create-cloudflare/**
+env:
+  node-version: 18.17.1
+  bun-version: 1.0.3
+jobs:
+  e2e:
+    name: ${{ format('E2E ({0})', matrix.pm) }}
+    if: |
+      github.event.pull_request.user.login != 'dependabot[bot]'
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        pm: [npm, pnpm, bun]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Install Dependencies
+        uses: ./.github/actions/install-dependencies
+
+      - name: E2E Tests
+        uses: ./.github/actions/run-c3-e2e
+        with:
+          package-manager: ${{ matrix.pm }}
+          quarantine: false
+          accountId: ${{ secrets.C3_TEST_CLOUDFLARE_ACCOUNT_ID }}
+          apiToken: ${{ secrets.C3_TEST_CLOUDFLARE_API_TOKEN }}