Merge branch 'main' into renovate/peculiar-asn1-x509-2.x

.github/workflows/ci.yml

@@ -198,7 +198,7 @@ jobs:
         run: npm run build
 
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     timeout-minutes: 30
 
@@ -209,7 +209,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:13.15
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
@@ -242,7 +242,9 @@ jobs:
       - run: npm run db:restore
       - run: npm run db:migrate
 
-      - run: npm run test -- --ignore "test/server/graphql/**"
+      # We have to specify OPENSSL_CONF=/dev/null for html-pdf to work
+      # We can remove it once we tackle https://github.com/opencollective/opencollective/issues/7622
+      - run: OPENSSL_CONF=/dev/null npm run test -- --ignore "test/server/graphql/**"
 
       - name: Report coverage
         uses: codecov/codecov-action@v4
@@ -251,7 +253,7 @@ jobs:
           flags: Unit
 
   test-graphql:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
 
     services:
@@ -261,15 +263,26 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:13.15
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
           POSTGRES_HOST_AUTH_METHOD: trust
         ports:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-
+      elastic:
+        image: elasticsearch:8.15.2
+        ports:
+          - 9200:9200
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        options: >-
+          --health-cmd="curl --silent --fail http://localhost:9200/_cluster/health || exit 1"
+          --health-interval=10s
+          --health-timeout=15s
+          --health-retries=30
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -313,7 +326,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:13.15
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
@@ -355,7 +368,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:13.15
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres

.github/workflows/e2e.yml

@@ -15,7 +15,7 @@ env:
 
   E2E_TEST: 1
   PGHOST: localhost
-  PGUSER: postgres
+  PGUSER: opencollective
   CYPRESS_RECORD: false
   CYPRESS_VIDEO: false
   CYPRESS_VIDEO_UPLOAD_ON_PASSES: false
@@ -34,12 +34,12 @@ env:
 
 jobs:
   e2e:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 30
 
     strategy:
       matrix:
-        files: ['0*.js', '1*.js', '2*.js', '3*.js']
+        files: ['0*.js', '1*.js', '2*.js', '3*.js', '4*.js']
 
     services:
       redis:
@@ -48,7 +48,7 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
       postgres:
-        image: postgres:13.15
+        image: postgres:16.6
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres
@@ -62,14 +62,15 @@ jobs:
       - name: Update apt
         run: sudo apt-get update || exit 0
 
-      - name: Install Cypress dependencies
-        run: sudo apt-get install --no-install-recommends -y libgtk2.0-0 libgtk-3-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb fonts-arphic-bkai00mp fonts-arphic-bsmi00lp fonts-arphic-gbsn00lp fonts-arphic-gkai00mp fonts-arphic-ukai fonts-arphic-uming ttf-wqy-zenhei ttf-wqy-microhei xfonts-wqy
-
-      - name: Install postgresql-client
-        run: sudo apt-get install -y postgresql-client
-
-      - name: Install graphicsmagick
-        run: sudo apt-get install -y graphicsmagick
+      - name: Install dependencies
+        run: |
+          sudo apt-get install \
+          `# Cypress dependencies - see https://docs.cypress.io/app/get-started/install-cypress#UbuntuDebian` \
+          libgtk2.0-0t64 libgtk-3-0t64 libgbm-dev libnotify-dev libnss3 libxss1 libasound2t64 libxtst6 xauth xvfb \
+          `# Postgres client` \
+          postgresql-client-16 \
+          `# GraphicsMagick (not sure if needed)` \
+          graphicsmagick
 
       - name: Install stripe-cli
         run: |
@@ -228,7 +229,7 @@ jobs:
 
       - name: Build (pdf)
         working-directory: opencollective-pdf
-        run: npm run build
+        run: OPENSSL_CONF=/dev/null npm run build
 
       # Setup Cypress
 

.gitignore

@@ -39,6 +39,7 @@ typings
 
 # test artifacts
 output/
+!test/mocks/files/*.csv
 
 *.swp
 

README.md

@@ -18,7 +18,7 @@ If you see a step below that could be improved (or is outdated), please update t
 
 2. Make sure you have a PostgreSQL database available
 
-- Check the version: 11.0, 10.3, 9.6.8, 9.5.12, 9.4.17, 9.3.22 or newer
+- Check the version: 14.x or newer
 - More info in our [PostgreSQL Database](docs/postgres.md) documentation
 
 3. For [node-gyp](https://github.com/nodejs/node-gyp), make sure you have Python 2 available and configured as the active version.

checks/model/collectives.js

@@ -25,8 +25,33 @@ async function checkDeletedUsers() {
   }
 }
 
+async function checkActiveApprovedAtInconsistency() {
+  const message = 'approvedAt and isActive are inconsistent (no auto fix)';
+
+  const [results] = await sequelize.query(
+    `
+    SELECT
+      COUNT(*) FILTER (WHERE "isActive" IS TRUE and "approvedAt" IS NULL) as "activeUnapproved",
+      COUNT(*) FILTER (WHERE "isActive" IS NOT TRUE and "approvedAt" IS NOT NULL) as "inactiveApproved"
+    FROM "Collectives"
+    WHERE "deletedAt" IS NULL
+    AND (
+      ("isActive" IS TRUE and "approvedAt" IS NULL)
+      OR ("isActive" IS NOT TRUE and "approvedAt" IS NOT NULL)
+    )`,
+    { type: sequelize.QueryTypes.SELECT, raw: true },
+  );
+
+  if (results.activeUnapproved > 0 || results.inactiveApproved > 0) {
+    throw new Error(
+      `${message} (${results.activeUnapproved} activeUnapproved, ${results.inactiveApproved} inactiveApproved)`,
+    );
+  }
+}
+
 export async function checkCollectives() {
   await checkDeletedUsers();
+  await checkActiveApprovedAtInconsistency();
 }
 
 if (!module.parent) {

checks/model/index.js

@@ -11,6 +11,7 @@ import { checkIndependentCollectives } from './independent-collectives';
 import { checkMembers } from './members';
 import { checkOrders } from './orders';
 import { checkPaymentMethods } from './payment-methods';
+import { checkTiers } from './tiers';
 import { checkTransactions } from './transactions';
 import { checkUsers } from './users';
 import { checkVirtualCards } from './virtual-cards';
@@ -23,6 +24,7 @@ const allModelChecks = [
   checkMembers,
   checkOrders,
   checkPaymentMethods,
+  checkTiers,
   checkTransactions,
   checkUsers,
   checkVirtualCards,

checks/model/members.js

@@ -1,9 +1,9 @@
 import '../../server/env';
 
-import { flatten, uniq } from 'lodash';
+import { flatten, min, uniq } from 'lodash';
 
 import logger from '../../server/lib/logger';
-import { MigrationLog, sequelize } from '../../server/models';
+import { Member, MigrationLog, sequelize } from '../../server/models';
 import { MigrationLogType } from '../../server/models/MigrationLog';
 
 import { runCheckThenExit } from './_utils';
@@ -70,7 +70,7 @@ async function checkDuplicateMembers({ fix = false } = {}) {
     `SELECT ARRAY_AGG(DISTINCT m2.id) AS duplicate_ids
      FROM "Members" m1
      INNER JOIN "Members" m2
-       ON m1.id != m2.id
+       ON m1.id < m2.id
        AND m1."CollectiveId" = m2."CollectiveId"
        AND m1."MemberCollectiveId" = m2."MemberCollectiveId"
        AND m1."role" = m2."role"
@@ -107,10 +107,73 @@ async function checkDuplicateMembers({ fix = false } = {}) {
   }
 }
 
+async function checkMissingMembers({ fix = false }) {
+  const message = 'No missing members';
+
+  const results = await sequelize.query(
+    `
+    SELECT
+      o."FromCollectiveId",
+      o."CollectiveId",
+      o."TierId",
+      t."type" AS "tierType",
+      ARRAY_AGG(o."CreatedByUserId") AS "CreatedByUserId",
+      ARRAY_AGG(o."createdAt") AS "createdAt"
+    FROM "Orders" o
+    LEFT JOIN "Members" m
+      ON o."FromCollectiveId" = m."MemberCollectiveId"
+      AND m."CollectiveId" = o."CollectiveId"
+      AND m."deletedAt" IS NULL
+      AND (
+        (m."TierId" IS NULL AND o."TierId" IS NULL)
+        OR (m."TierId" = o."TierId")
+      )
+    INNER JOIN "Collectives" c
+      ON c."id" = o."CollectiveId" AND c."deletedAt" IS NULL
+    INNER JOIN "Collectives" fc
+      ON fc."id" = o."FromCollectiveId" AND fc."deletedAt" IS NULL
+    LEFT JOIN "Tiers" t
+      ON t."id" = o."TierId" AND t."deletedAt" IS NULL
+    WHERE o.status in ('PAID', 'ACTIVE')
+    AND o."deletedAt" IS NULL
+    AND m.id IS NULL
+    GROUP BY o."FromCollectiveId", o."CollectiveId", o."TierId", t."type"
+    `,
+    { type: sequelize.QueryTypes.SELECT, raw: true },
+  );
+
+  if (results.length > 0) {
+    if (!fix) {
+      throw new Error(message);
+    } else {
+      logger.warn(`Fixing: ${message}`);
+      for (const result of results) {
+        await Member.create({
+          MemberCollectiveId: result.FromCollectiveId,
+          CollectiveId: result.CollectiveId,
+          CreatedByUserId: result.CreatedByUserId[0],
+          TierId: result.TierId,
+          since: min(result.createdAt),
+          role: result.tierType === 'TICKET' ? 'ATTENDEE' : 'BACKER',
+        });
+      }
+
+      // Members don't have a `data` column that we could use to log that they've been created from this script, so we
+      // create a new migration log instead.
+      await MigrationLog.create({
+        type: MigrationLogType.MODEL_FIX,
+        description: `Missing members check: Added ${results.length} missing members`,
+        data: { results },
+      });
+    }
+  }
+}
+
 export async function checkMembers({ fix = false } = {}) {
   await checkDeletedMembers({ fix });
   await checkMemberTypes();
   await checkDuplicateMembers({ fix });
+  await checkMissingMembers({ fix });
 }
 
 if (!module.parent) {

checks/model/orders.js

@@ -29,8 +29,37 @@ async function checkDuplicateNonRecurringContribution() {
   }
 }
 
-export async function checkOrders() {
+async function checkPaidOrdersWithNullProcessedAt({ fix = false } = {}) {
+  const message = 'Paid Order with null processedAt';
+
+  const results = await sequelize.query(
+    `
+    SELECT id, "updatedAt"
+    FROM "Orders"
+    WHERE status = 'PAID'
+    AND "processedAt" IS NULL
+    ORDER BY "createdAt" DESC
+  `,
+    { type: sequelize.QueryTypes.SELECT, raw: true },
+  );
+
+  if (results.length > 0) {
+    if (!fix) {
+      throw new Error(message);
+    } else {
+      await sequelize.query(`
+        UPDATE "Orders"
+        SET "processedAt" = "updatedAt"
+        WHERE status = 'PAID'
+        AND "processedAt" IS NULL
+      `);
+    }
+  }
+}
+
+export async function checkOrders({ fix = false } = {}) {
   await checkDuplicateNonRecurringContribution();
+  await checkPaidOrdersWithNullProcessedAt({ fix });
 }
 
 if (!module.parent) {

checks/model/tiers.ts

@@ -0,0 +1,43 @@
+import '../../server/env';
+
+import { sequelize } from '../../server/models';
+
+import { runCheckThenExit } from './_utils';
+
+async function checkTiersMinimumAmountWithPresets({ fix = false } = {}) {
+  const message = 'Tiers presets cannot be lower than the minimum amount';
+  const results = await sequelize.query(
+    `
+    SELECT COUNT(*) AS count
+    FROM "Tiers"
+    WHERE presets IS NOT NULL
+    AND ARRAY_LENGTH(presets, 1) > 0
+    AND "minimumAmount" > (SELECT MIN(val) FROM UNNEST(presets) val)  
+  `,
+    { type: sequelize.QueryTypes.SELECT, raw: true },
+  );
+
+  if (results[0].count > 0) {
+    if (!fix) {
+      throw new Error(`${message} (${results[0].count} found)`);
+    }
+
+    await sequelize.query(`
+      UPDATE "Tiers"
+      SET
+        "minimumAmount" = (SELECT MIN(val) FROM UNNEST(presets) val),
+        "data" = JSONB_SET(COALESCE("data", '{}'), '{minimumAmountBeforeCheckFix}', "minimumAmount"::TEXT::JSONB)
+      WHERE presets IS NOT NULL
+      AND ARRAY_LENGTH(presets, 1) > 0
+      AND "minimumAmount" > (SELECT MIN(val) FROM UNNEST(presets) val)
+    `);
+  }
+}
+
+export async function checkTiers({ fix = false } = {}) {
+  await checkTiersMinimumAmountWithPresets({ fix });
+}
+
+if (!module.parent) {
+  runCheckThenExit(checkTiers);
+}