From 6a933b2848bf265b05c2a30f5260ac6266717154 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Thu, 19 Oct 2023 23:55:55 +0200 Subject: [PATCH 01/17] chore(tests): Migrate MySQL database tests from drone to GitHub Signed-off-by: Ferdinand Thiessen --- .drone.yml | 39 -------- .github/workflows/phpunit-mysql.yml | 147 ++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/phpunit-mysql.yml diff --git a/.drone.yml b/.drone.yml index 4503c9a8741a3..de5014ea6398d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -384,45 +384,6 @@ trigger: - pull_request - push ---- -kind: pipeline -name: mysql8.0-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: mysql-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh mysql - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: mysql - image: ghcr.io/nextcloud/continuous-integration-mysql-8.0:mysql-8.0 - command: - - --default-authentication-plugin=mysql_native_password - - --sql-mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION - environment: - MYSQL_ROOT_PASSWORD: owncloud - MYSQL_USER: oc_autotest - MYSQL_PASSWORD: owncloud - MYSQL_DATABASE: oc_autotest - tmpfs: - - /var/lib/mysql - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - --- kind: pipeline name: postgres10-php8.0 diff --git a/.github/workflows/phpunit-mysql.yml b/.github/workflows/phpunit-mysql.yml new file mode 100644 index 0000000000000..c9f5b103dd962 --- /dev/null +++ b/.github/workflows/phpunit-mysql.yml @@ -0,0 +1,147 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization + +name: PHPUnit mysql + +on: + pull_request: + push: + branches: + - main + - master + - stable* + +permissions: + contents: read + +concurrency: + group: phpunit-mysql-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-mysql: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + matrix: + php-versions: ['8.0'] + mysql-versions: ['8.0', '8.1'] + include: + - php-versions: '8.3' + mysql-versions: '8.1' + + name: MySQL ${{ matrix.mysql-versions }} (PHP ${{ matrix.php-versions }}) - database tests + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + mysql: + image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: oc_autotest + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 + + steps: + - name: Checkout server + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql + coverage: none + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Wait for MySQL + id: check_mysql_up + run: | + apps/files_external/tests/env/wait-for-connection localhost 4444 30 + + - name: Enable ONLY_FULL_GROUP_BY MySQL option + if: steps.check_mysql_up.outcome == 'success' + run: | + echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + + - name: Set up Nextcloud + if: steps.check_mysql_up.outcome == 'success' + env: + DB_PORT: 4444 + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit + if: steps.check_mysql_up.outcome == 'success' + run: composer run test:db + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest + needs: [changes, phpunit-mysql] + + if: always() + + name: phpunit-mysql-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-mysql.result != 'success' }}; then exit 1; fi From 4e7ff6d7f7df141a1d6a33df73f1cb1c5a335bf3 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Fri, 20 Oct 2023 00:45:47 +0200 Subject: [PATCH 02/17] chore(tests): Migrate `mariadb` tests from drone to GitHub Signed-off-by: Ferdinand Thiessen --- .drone.yml | 114 -------------------- .github/workflows/phpunit-mariadb.yml | 147 ++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 114 deletions(-) create mode 100644 .github/workflows/phpunit-mariadb.yml diff --git a/.drone.yml b/.drone.yml index de5014ea6398d..2194eec6b384a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -270,120 +270,6 @@ trigger: - pull_request - push ---- -kind: pipeline -name: mariadb10.2-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: mariadb10.2-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh mariadb - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: mariadb - image: ghcr.io/nextcloud/continuous-integration-mariadb-10.2:10.2 - environment: - MYSQL_ROOT_PASSWORD: owncloud - MYSQL_USER: oc_autotest - MYSQL_PASSWORD: owncloud - MYSQL_DATABASE: oc_autotest - command: - - --sql-mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION - tmpfs: - - /var/lib/mysql - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - ---- -kind: pipeline -name: mariadb10.4-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: mariadb10.4-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh mariadb - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: mariadb - image: ghcr.io/nextcloud/continuous-integration-mariadb-10.4:10.4 - environment: - MYSQL_ROOT_PASSWORD: owncloud - MYSQL_USER: oc_autotest - MYSQL_PASSWORD: owncloud - MYSQL_DATABASE: oc_autotest - command: - - --sql-mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION - tmpfs: - - /var/lib/mysql - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - ---- -kind: pipeline -name: mariadb10.6-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: mariadb10.6-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh mariadb - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: mariadb - image: ghcr.io/nextcloud/continuous-integration-mariadb-10.6:latest - environment: - MYSQL_ROOT_PASSWORD: owncloud - MYSQL_USER: oc_autotest - MYSQL_PASSWORD: owncloud - MYSQL_DATABASE: oc_autotest - command: - - --sql-mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION - tmpfs: - - /var/lib/mysql - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - --- kind: pipeline name: postgres10-php8.0 diff --git a/.github/workflows/phpunit-mariadb.yml b/.github/workflows/phpunit-mariadb.yml new file mode 100644 index 0000000000000..4ba37751d280f --- /dev/null +++ b/.github/workflows/phpunit-mariadb.yml @@ -0,0 +1,147 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization + +name: PHPUnit mariadb + +on: + pull_request: + push: + branches: + - main + - master + - stable* + +permissions: + contents: read + +concurrency: + group: phpunit-mariadb-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-mariadb: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + matrix: + php-versions: ['8.0'] + mariadb-versions: ['10.3', '10.4', '10.5', '10.6', '10.11'] + include: + - php-versions: '8.3' + mariadb-versions: '10.6' + + name: MariaDB ${{ matrix.mariadb-versions }} (PHP ${{ matrix.php-versions }}) - database tests + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + mariadb: + image: mariadb:${{ matrix.mariadb-versions }} + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: oc_autotest + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 + + steps: + - name: Checkout server + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql + coverage: none + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Wait for MariaDB + id: check_mariadb_up + run: | + apps/files_external/tests/env/wait-for-connection localhost 4444 30 + + - name: Enable ONLY_FULL_GROUP_BY MySQL option + if: steps.check_mariadb_up.outcome == 'success' + run: | + echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + + - name: Set up Nextcloud + if: steps.check_mariadb_up.outcome == 'success' + env: + DB_PORT: 4444 + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit + if: steps.check_mariadb_up.outcome == 'success' + run: composer run test:db + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest + needs: [changes, phpunit-mariadb] + + if: always() + + name: phpunit-mariadb-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-mariadb.result != 'success' }}; then exit 1; fi From bb4f108488b48b2241d424993eec87b3cbf09bb7 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Fri, 20 Oct 2023 01:00:09 +0200 Subject: [PATCH 03/17] chore(tests): Migrate `postgres` tests from drone to GitHub Signed-off-by: Ferdinand Thiessen --- .drone.yml | 144 --------------------------- .github/workflows/phpunit-pgsql.yml | 146 ++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 144 deletions(-) create mode 100644 .github/workflows/phpunit-pgsql.yml diff --git a/.drone.yml b/.drone.yml index 2194eec6b384a..1f73c3a35ec1f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -270,150 +270,6 @@ trigger: - pull_request - push ---- -kind: pipeline -name: postgres10-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: postgres-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - sleep 10 # gives the database enough time to initialize - - POSTGRES=10 NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh pgsql - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: postgres-10 - image: ghcr.io/nextcloud/continuous-integration-postgres-10:postgres-10 - environment: - POSTGRES_USER: oc_autotest - POSTGRES_DB: oc_autotest - POSTGRES_PASSWORD: owncloud - tmpfs: - - /var/lib/postgresql/data - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - ---- -kind: pipeline -name: postgres11-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: postgres-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - sleep 10 # gives the database enough time to initialize - - POSTGRES=11 NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh pgsql - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: postgres-11 - image: ghcr.io/nextcloud/continuous-integration-postgres-11:postgres-11 - environment: - POSTGRES_USER: oc_autotest - POSTGRES_DB: oc_autotest - POSTGRES_PASSWORD: owncloud - tmpfs: - - /var/lib/postgresql/data - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - ---- -kind: pipeline -name: postgres13-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: postgres-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - sleep 10 # gives the database enough time to initialize - - POSTGRES=13 NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh pgsql - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: postgres-13 - image: ghcr.io/nextcloud/continuous-integration-postgres-13:postgres-13 - environment: - POSTGRES_USER: oc_autotest - POSTGRES_DB: oc_autotest - POSTGRES_PASSWORD: owncloud - tmpfs: - - /var/lib/postgresql/data - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - ---- -kind: pipeline -name: postgres15-php8.0 - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: postgres-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - sleep 10 # gives the database enough time to initialize - - POSTGRES=15 NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh pgsql - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest -- name: postgres-15 - image: ghcr.io/nextcloud/continuous-integration-postgres-15:latest - environment: - POSTGRES_USER: oc_autotest - POSTGRES_DB: oc_autotest - POSTGRES_PASSWORD: owncloud - tmpfs: - - /var/lib/postgresql/data - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - --- kind: pipeline name: nodb-codecov diff --git a/.github/workflows/phpunit-pgsql.yml b/.github/workflows/phpunit-pgsql.yml new file mode 100644 index 0000000000000..bebf8a092826b --- /dev/null +++ b/.github/workflows/phpunit-pgsql.yml @@ -0,0 +1,146 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization + +name: PHPUnit pgsql + +on: + pull_request: + push: + branches: + - main + - master + - stable* + +permissions: + contents: read + +concurrency: + group: phpunit-pgsql-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-pgsql: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + matrix: + php-versions: ['8.0'] + # To keep the matrix smaller we ignore PostgreSQL '11', '13', '14' as we already test 10 and 15 as lower and upper bound + postgres-versions: ['10', '15', '16'] + include: + - php-versions: '8.3' + postgres-versions: '15' + + name: PostgreSQL ${{ matrix.postgres-versions }} (PHP ${{ matrix.php-versions }}) - database tests + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + postgres: + image: ghcr.io/nextcloud/continuous-integration-postgres-${{ matrix.postgres-versions }}:latest + ports: + - 4444:5432/tcp + env: + POSTGRES_USER: root + POSTGRES_PASSWORD: rootpassword + POSTGRES_DB: nextcloud + options: --mount type=tmpfs,destination=/var/lib/postgresql/data --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 + + steps: + - name: Checkout server + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, pgsql, pdo_pgsql + coverage: none + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Wait for pgsql + id: check_pgsql_up + run: | + apps/files_external/tests/env/wait-for-connection localhost 4444 30 + + - name: Set up Nextcloud + if: steps.check_pgsql_up.outcome == 'success' + env: + DB_PORT: 4444 + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit + if: steps.check_pgsql_up.outcome == 'success' + run: composer run test:db + + - name: Run repair steps + if: steps.check_pgsql_up.outcome == 'success' + run: | + ./occ maintenance:repair --include-expensive + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest + needs: [changes, phpunit-pgsql] + + if: always() + + name: phpunit-pgsql-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-pgsql.result != 'success' }}; then exit 1; fi From 32a48497578aad4c9bbebd961cf4785c5ec22cec Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Fri, 20 Oct 2023 01:47:09 +0200 Subject: [PATCH 04/17] chore(tests): Migrate `sqlite` tests from drone to GitHub Signed-off-by: Ferdinand Thiessen --- .drone.yml | 54 ------------ .github/workflows/phpunit-sqlite.yml | 126 +++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/phpunit-sqlite.yml diff --git a/.drone.yml b/.drone.yml index 1f73c3a35ec1f..f4cab8f8bfe9b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -216,60 +216,6 @@ trigger: - pull_request - push ---- -kind: pipeline -name: nodb - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: nodb-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - NOCOVERAGE=true TEST_SELECTION=NODB ./autotest.sh sqlite - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - ---- -kind: pipeline -name: sqlite - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: sqlite-php8.0 - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - commands: - - bash tests/drone-run-php-tests.sh || exit 0 - - NOCOVERAGE=true TEST_SELECTION=DB ./autotest.sh sqlite - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - --- kind: pipeline name: nodb-codecov diff --git a/.github/workflows/phpunit-sqlite.yml b/.github/workflows/phpunit-sqlite.yml new file mode 100644 index 0000000000000..afdb0beaa56eb --- /dev/null +++ b/.github/workflows/phpunit-sqlite.yml @@ -0,0 +1,126 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization + +name: PHPUnit sqlite + +on: + pull_request: + push: + branches: + - main + - master + - stable* + +permissions: + contents: read + +concurrency: + group: phpunit-sqlite-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-sqlite: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + database-only: [true] + include: + - php-versions: '8.2' + database-only: false + + name: SQLite (PHP ${{ matrix.php-versions }}) - ${{ matrix.database-only && 'database' || 'all' }} tests + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + steps: + - name: Checkout server + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite ${{ matrix.database-only && '' || ', imagick' }} + coverage: none + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Set up Nextcloud + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit database tests + if: ${{ matrix.database-only }} + run: composer run test:db + + - name: PHPUnit full testsuite + if: ${{ ! matrix.database-only }} + run: composer run test + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest + needs: [changes, phpunit-sqlite] + + if: always() + + name: phpunit-sqlite-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-sqlite.result != 'success' }}; then exit 1; fi From 8a682328047bd9201f0b868b11f077786e239eee Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Fri, 20 Oct 2023 02:09:27 +0200 Subject: [PATCH 05/17] chore(tests): Migrate code coverage generation from drone to GitHub Signed-off-by: Ferdinand Thiessen --- .drone.yml | 64 ------------- .github/workflows/phpunit-mariadb.yml | 20 ++-- .github/workflows/phpunit-mysql.yml | 24 ++--- .github/workflows/phpunit-nodb.yml | 130 ++++++++++++++++++++++++++ .github/workflows/phpunit-pgsql.yml | 22 ++--- .github/workflows/phpunit-sqlite.yml | 28 +++--- 6 files changed, 179 insertions(+), 109 deletions(-) create mode 100644 .github/workflows/phpunit-nodb.yml diff --git a/.drone.yml b/.drone.yml index f4cab8f8bfe9b..06564ee90c7ef 100644 --- a/.drone.yml +++ b/.drone.yml @@ -216,70 +216,6 @@ trigger: - pull_request - push ---- -kind: pipeline -name: nodb-codecov - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: nodb-codecov - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - environment: - CODECOV_TOKEN: - from_secret: CODECOV_TOKEN - XDEBUG_MODE: coverage - commands: - - phpenmod xdebug - - TEST_SELECTION=NODB ./autotest.sh sqlite - - wget https://codecov.io/bash -O codecov.sh - - bash codecov.sh -Z -C $DRONE_COMMIT -f tests/autotest-clover-sqlite.xml - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest - -trigger: - branch: - - master - - stable* - event: - - push - ---- -kind: pipeline -name: db-codecov - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: db-codecov - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - environment: - CODECOV_TOKEN: - from_secret: CODECOV_TOKEN - XDEBUG_MODE: coverage - commands: - - phpenmod xdebug - - TEST_SELECTION=QUICKDB ./autotest.sh sqlite - - wget https://codecov.io/bash -O codecov.sh - - bash codecov.sh -Z -C $DRONE_COMMIT -f tests/autotest-clover-sqlite.xml - -services: -- name: cache - image: ghcr.io/nextcloud/continuous-integration-redis:latest - -trigger: - branch: - - master - - stable* - event: - - push - --- kind: pipeline name: object-store-s3 diff --git a/.github/workflows/phpunit-mariadb.yml b/.github/workflows/phpunit-mariadb.yml index 4ba37751d280f..cf213d1c1b457 100644 --- a/.github/workflows/phpunit-mariadb.yml +++ b/.github/workflows/phpunit-mariadb.yml @@ -60,6 +60,7 @@ jobs: include: - php-versions: '8.3' mariadb-versions: '10.6' + coverage: true name: MariaDB ${{ matrix.mariadb-versions }} (PHP ${{ matrix.php-versions }}) - database tests @@ -93,7 +94,7 @@ jobs: php-version: ${{ matrix.php-versions }} # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql - coverage: none + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} ini-file: development env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -101,19 +102,12 @@ jobs: - name: Set up dependencies run: composer i - - name: Wait for MariaDB - id: check_mariadb_up - run: | - apps/files_external/tests/env/wait-for-connection localhost 4444 30 - - name: Enable ONLY_FULL_GROUP_BY MySQL option - if: steps.check_mariadb_up.outcome == 'success' run: | echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword - name: Set up Nextcloud - if: steps.check_mariadb_up.outcome == 'success' env: DB_PORT: 4444 run: | @@ -124,8 +118,14 @@ jobs: php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 - name: PHPUnit - if: steps.check_mariadb_up.outcome == 'success' - run: composer run test:db + run: composer run test:db ${{ matrix.coverage && ' -- --coverage-clover ./clover.db.xml' || '' }} + + - name: Upload db code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.db.xml + flags: phpunit-mariadb - name: Print logs if: always() diff --git a/.github/workflows/phpunit-mysql.yml b/.github/workflows/phpunit-mysql.yml index c9f5b103dd962..1c47951360341 100644 --- a/.github/workflows/phpunit-mysql.yml +++ b/.github/workflows/phpunit-mysql.yml @@ -58,8 +58,9 @@ jobs: php-versions: ['8.0'] mysql-versions: ['8.0', '8.1'] include: - - php-versions: '8.3' - mysql-versions: '8.1' + - mysql-versions: '8.1' + php-versions: '8.3' + coverage: true name: MySQL ${{ matrix.mysql-versions }} (PHP ${{ matrix.php-versions }}) - database tests @@ -93,7 +94,7 @@ jobs: php-version: ${{ matrix.php-versions }} # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql - coverage: none + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} ini-file: development env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -101,19 +102,12 @@ jobs: - name: Set up dependencies run: composer i - - name: Wait for MySQL - id: check_mysql_up - run: | - apps/files_external/tests/env/wait-for-connection localhost 4444 30 - - name: Enable ONLY_FULL_GROUP_BY MySQL option - if: steps.check_mysql_up.outcome == 'success' run: | echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword - name: Set up Nextcloud - if: steps.check_mysql_up.outcome == 'success' env: DB_PORT: 4444 run: | @@ -124,8 +118,14 @@ jobs: php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 - name: PHPUnit - if: steps.check_mysql_up.outcome == 'success' - run: composer run test:db + run: composer run test:db ${{ matrix.coverage && ' -- --coverage-clover ./clover.db.xml' || '' }} + + - name: Upload db code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.db.xml + flags: phpunit-mysql - name: Print logs if: always() diff --git a/.github/workflows/phpunit-nodb.yml b/.github/workflows/phpunit-nodb.yml new file mode 100644 index 0000000000000..b9dea3c28ee58 --- /dev/null +++ b/.github/workflows/phpunit-nodb.yml @@ -0,0 +1,130 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization + +# This is the testsuite running all non-database agnostic unit tests +name: PHPUnit nodb + +on: + pull_request: + push: + branches: + - main + - master + - stable* + +permissions: + contents: read + +concurrency: + group: phpunit-nodb-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-nodb: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.2' + coverage: true + + name: No DB unit tests (PHP ${{ matrix.php-versions }}) + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + steps: + - name: Checkout server + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@7fdd3ece872ec7ec4c098ae5ab7637d5e0a96067 # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, imagick, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + # Required for tests that use pcntl + ini-values: disable_functions="" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Set up Nextcloud + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit nodb testsuite + run: composer run test -- --exclude-group DB,SLOWDB ${{ matrix.coverage && ' --coverage-clover ./clover.nodb.xml' || '' }} + + - name: Upload nodb code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.nodb.xml + flags: phpunit-nodb + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest + needs: [changes, phpunit-nodb] + + if: always() + + name: phpunit-nodb-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-nodb.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/phpunit-pgsql.yml b/.github/workflows/phpunit-pgsql.yml index bebf8a092826b..ddcd20ed51ccd 100644 --- a/.github/workflows/phpunit-pgsql.yml +++ b/.github/workflows/phpunit-pgsql.yml @@ -61,6 +61,7 @@ jobs: include: - php-versions: '8.3' postgres-versions: '15' + coverage: true name: PostgreSQL ${{ matrix.postgres-versions }} (PHP ${{ matrix.php-versions }}) - database tests @@ -93,7 +94,7 @@ jobs: php-version: ${{ matrix.php-versions }} # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, pgsql, pdo_pgsql - coverage: none + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} ini-file: development env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -101,13 +102,7 @@ jobs: - name: Set up dependencies run: composer i - - name: Wait for pgsql - id: check_pgsql_up - run: | - apps/files_external/tests/env/wait-for-connection localhost 4444 30 - - name: Set up Nextcloud - if: steps.check_pgsql_up.outcome == 'success' env: DB_PORT: 4444 run: | @@ -117,12 +112,17 @@ jobs: ./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 - - name: PHPUnit - if: steps.check_pgsql_up.outcome == 'success' - run: composer run test:db + - name: PHPUnit database tests + run: composer run test:db ${{ matrix.coverage && ' -- --coverage-clover ./clover.db.xml' || '' }} + + - name: Upload db code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.db.xml + flags: phpunit-postgres - name: Run repair steps - if: steps.check_pgsql_up.outcome == 'success' run: | ./occ maintenance:repair --include-expensive diff --git a/.github/workflows/phpunit-sqlite.yml b/.github/workflows/phpunit-sqlite.yml index afdb0beaa56eb..61710f31b6fcd 100644 --- a/.github/workflows/phpunit-sqlite.yml +++ b/.github/workflows/phpunit-sqlite.yml @@ -56,12 +56,11 @@ jobs: strategy: matrix: php-versions: ['8.0', '8.1', '8.2', '8.3'] - database-only: [true] include: - - php-versions: '8.2' - database-only: false + - php-versions: '8.1' + coverage: true - name: SQLite (PHP ${{ matrix.php-versions }}) - ${{ matrix.database-only && 'database' || 'all' }} tests + name: SQLite (PHP ${{ matrix.php-versions }}) services: cache: @@ -77,12 +76,12 @@ jobs: submodules: true - name: Set up php ${{ matrix.php-versions }} - uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + uses: shivammathur/setup-php@7fdd3ece872ec7ec4c098ae5ab7637d5e0a96067 # v2 with: php-version: ${{ matrix.php-versions }} # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation - extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite ${{ matrix.database-only && '' || ', imagick' }} - coverage: none + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} ini-file: development env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -98,13 +97,18 @@ jobs: ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + - name: Nextcloud debug information + run: ./occ app:list && echo "======= System config =======" && ./occ config:list system + - name: PHPUnit database tests - if: ${{ matrix.database-only }} - run: composer run test:db + run: composer run test:db ${{ matrix.coverage && ' -- --coverage-clover ./clover.db.xml' || '' }} - - name: PHPUnit full testsuite - if: ${{ ! matrix.database-only }} - run: composer run test + - name: Upload db code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.db.xml + flags: phpunit-sqlite - name: Print logs if: always() From a630ee8dceb306f2b491bbc5f92d3cf2f28c641d Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Fri, 20 Oct 2023 11:36:56 +0200 Subject: [PATCH 06/17] chore(tests): Align oracle database tests with the other databases Signed-off-by: Ferdinand Thiessen --- .github/workflows/phpunit-oci.yml | 60 ++++++++++++++++++++++--------- tests/redis.config.php | 2 +- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/.github/workflows/phpunit-oci.yml b/.github/workflows/phpunit-oci.yml index 2c90b992cd073..fb2081876a75d 100644 --- a/.github/workflows/phpunit-oci.yml +++ b/.github/workflows/phpunit-oci.yml @@ -1,7 +1,6 @@ name: PHPUnit oci -on: - pull_request: +on: pull_request permissions: contents: read @@ -26,36 +25,51 @@ jobs: src: - '.github/workflows/**' - '3rdparty/**' + - '**/appinfo/**' - '**/lib/**' + - '**/templates/**' - '**/tests/**' - - '**/vendor-bin/**' + - 'vendor/**' + - 'vendor-bin/**' - '.php-cs-fixer.dist.php' - 'composer.json' - 'composer.lock' - '**.php' phpunit-oci: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest if: ${{ github.repository_owner != 'nextcloud-gmbh' }} strategy: matrix: - php-versions: ['8.0', '8.1', '8.2'] + oracle-versions: ['11'] + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.3' + coverage: true + + name: Oracle ${{ matrix.oracle-versions }} (PHP ${{ matrix.php-versions }}) - database tests services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + oracle: - image: ghcr.io/gvenzl/oracle-xe:11 + image: ghcr.io/gvenzl/oracle-xe:${{ matrix.oracle-versions }} # Provide passwords and other environment variables to container env: ORACLE_RANDOM_PASSWORD: true - APP_USER: autotest - APP_USER_PASSWORD: owncloud + APP_USER: oc_autotest + APP_USER_PASSWORD: nextcloud # Forward Oracle port ports: - - 1521:1521/tcp + - 4444:1521/tcp # Provide healthcheck script options for startup options: >- @@ -71,24 +85,38 @@ jobs: submodules: true - name: Set up php ${{ matrix.php-versions }} - uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 with: php-version: ${{ matrix.php-versions }} - extensions: ctype, curl, dom, fileinfo, gd, imagick, intl, json, mbstring, oci8, openssl, pcntl, pdo_sqlite, posix, sqlite, xml, zip - coverage: none + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, oci8 + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} ini-file: development env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set up dependencies + run: composer i + - name: Set up Nextcloud + env: + DB_PORT: 4444 run: | - composer install mkdir data - ./occ maintenance:install --verbose --database=oci --database-name=XE --database-host=127.0.0.1 --database-port=1521 --database-user=autotest --database-pass=owncloud --admin-user admin --admin-pass admin - php -f index.php + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=oci --database-name=XE --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=oc_autotest --database-pass=nextcloud --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 - name: PHPUnit - run: composer run test:db + run: composer run test:db ${{ matrix.coverage && ' -- --coverage-clover ./clover.db.xml' || '' }} + + - name: Upload db code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.db.xml + flags: phpunit-oci - name: Run repair steps run: | diff --git a/tests/redis.config.php b/tests/redis.config.php index 9d3f1eca63eac..2ff46ec672815 100644 --- a/tests/redis.config.php +++ b/tests/redis.config.php @@ -5,7 +5,7 @@ 'memcache.distributed' => '\\OC\\Memcache\\Redis', 'memcache.locking' => '\\OC\\Memcache\\Redis', 'redis' => [ - 'host' => 'cache', + 'host' => 'localhost', 'port' => 6379, 'timeout' => 0, ], From b8fbd7721d25f9863d452db91fb3823108641dfa Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sat, 21 Oct 2023 04:00:24 +0200 Subject: [PATCH 07/17] fix(user_ldap): Ensure host is a string when checking if `ldapi` Signed-off-by: Ferdinand Thiessen --- apps/user_ldap/lib/Configuration.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/user_ldap/lib/Configuration.php b/apps/user_ldap/lib/Configuration.php index 43d7b5cfbf1d4..b009ba3f4809e 100644 --- a/apps/user_ldap/lib/Configuration.php +++ b/apps/user_ldap/lib/Configuration.php @@ -607,6 +607,7 @@ public function getAvatarAttributes(): array { * Returns TRUE if the ldapHost variable starts with 'ldapi://' */ public function usesLdapi(): bool { - return (substr($this->config['ldapHost'], 0, strlen('ldapi://')) === 'ldapi://'); + $host = $this->config['ldapHost']; + return is_string($host) && (substr($host, 0, strlen('ldapi://')) === 'ldapi://'); } } From c3d63c57ce741ef1ad9dad7b0217533141061c79 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 22 Oct 2023 01:16:53 +0200 Subject: [PATCH 08/17] fix(encryption): Fix deprecated syntax in encryption tests Signed-off-by: Ferdinand Thiessen --- apps/encryption/tests/Crypto/EncryptAllTest.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/encryption/tests/Crypto/EncryptAllTest.php b/apps/encryption/tests/Crypto/EncryptAllTest.php index 459f680a374ca..7c7e6a7f7ab4e 100644 --- a/apps/encryption/tests/Crypto/EncryptAllTest.php +++ b/apps/encryption/tests/Crypto/EncryptAllTest.php @@ -73,6 +73,9 @@ class EncryptAllTest extends TestCase { /** @var \PHPUnit\Framework\MockObject\MockObject | \OCP\IL10N */ protected $l; + /** @var \PHPUnit\Framework\MockObject\MockObject | IFactory */ + protected $l10nFactory; + /** @var \PHPUnit\Framework\MockObject\MockObject | \Symfony\Component\Console\Helper\QuestionHelper */ protected $questionHelper; @@ -119,7 +122,10 @@ protected function setUp(): void { $this->userInterface = $this->getMockBuilder(UserInterface::class) ->disableOriginalConstructor()->getMock(); - /* We need format method to return a string */ + /** + * We need format method to return a string + * @var OutputFormatterInterface|\PHPUnit\Framework\MockObject\MockObject + */ $outputFormatter = $this->createMock(OutputFormatterInterface::class); $outputFormatter->method('isDecorated')->willReturn(false); $outputFormatter->method('format')->willReturnArgument(0); From fceb781058516e7bd534719957a3d8354517b333 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 29 Oct 2023 01:38:08 +0200 Subject: [PATCH 09/17] fix(tests): Fix deprecation errors in S3 tests Signed-off-by: Ferdinand Thiessen --- tests/lib/Files/ObjectStore/S3Test.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php index fd451dc3c016f..33130a152a1e1 100644 --- a/tests/lib/Files/ObjectStore/S3Test.php +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -106,6 +106,7 @@ public function testSeek() { } public function assertNoUpload($objectUrn) { + /** @var \OC\Files\ObjectStore\S3 */ $s3 = $this->getInstance(); $s3client = $s3->getConnection(); $uploads = $s3client->listMultipartUploads([ @@ -119,7 +120,7 @@ public function testEmptyUpload() { $s3 = $this->getInstance(); $emptyStream = fopen("php://memory", "r"); - fwrite($emptyStream, null); + fwrite($emptyStream, ''); $s3->writeObject('emptystream', $emptyStream); From 8e02014b559dd310dd42cc3f5cf1e4097342d459 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 29 Oct 2023 01:16:46 +0200 Subject: [PATCH 10/17] chore(tests): Migrate object storage unit tests from drone to GitHub Actions Signed-off-by: Ferdinand Thiessen --- .drone.yml | 75 ----------- .github/workflows/object-storage-azure.yml | 133 ++++++++++++++++++++ .github/workflows/object-storage-s3.yml | 139 +++++++++++++++++++++ .github/workflows/object-storage-swift.yml | 129 +++++++++++++++++++ .github/workflows/s3-primary.yml | 87 ------------- tests/lib/Files/ObjectStore/SwiftTest.php | 4 + tests/preseed-config.php | 89 ++++++------- 7 files changed, 440 insertions(+), 216 deletions(-) create mode 100644 .github/workflows/object-storage-azure.yml create mode 100644 .github/workflows/object-storage-s3.yml create mode 100644 .github/workflows/object-storage-swift.yml delete mode 100644 .github/workflows/s3-primary.yml diff --git a/.drone.yml b/.drone.yml index 06564ee90c7ef..4cc000c628341 100644 --- a/.drone.yml +++ b/.drone.yml @@ -216,81 +216,6 @@ trigger: - pull_request - push ---- -kind: pipeline -name: object-store-s3 - -steps: -- name: minio - image: ghcr.io/nextcloud/continuous-integration-minio:latest - detach: true - commands: - - mkdir /s3data - - minio server /s3data - environment: - MINIO_ROOT_USER: nextcloud - MINIO_ROOT_PASSWORD: nextcloud -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: object-store - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - environment: - OBJECT_STORE: s3 - CODECOV_TOKEN: - from_secret: CODECOV_TOKEN - commands: - - phpenmod xdebug - - ./tests/drone-wait-objectstore.sh - - TEST_SELECTION=PRIMARY-s3 ./autotest.sh sqlite - - wget https://codecov.io/bash -O codecov.sh - - bash codecov.sh -C $DRONE_COMMIT -f tests/autotest-clover-sqlite.xml - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - ---- -kind: pipeline -name: object-store-azure - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: object-store - image: ghcr.io/nextcloud/continuous-integration-php8.0:latest - environment: - OBJECT_STORE: azure - CODECOV_TOKEN: - from_secret: CODECOV_TOKEN - commands: - - phpenmod xdebug - - ./tests/drone-wait-objectstore.sh - - TEST_SELECTION=PRIMARY-azure ./autotest.sh sqlite - - wget https://codecov.io/bash -O codecov.sh - - bash codecov.sh -C $DRONE_COMMIT -f tests/autotest-clover-sqlite.xml - -services: -- name: azurite - image: ghcr.io/nextcloud/continuous-integration-azurite:latest - environment: - executable: blob - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push - --- kind: pipeline name: memcache-memcached diff --git a/.github/workflows/object-storage-azure.yml b/.github/workflows/object-storage-azure.yml new file mode 100644 index 0000000000000..72389c5a816a2 --- /dev/null +++ b/.github/workflows/object-storage-azure.yml @@ -0,0 +1,133 @@ +name: Object storage azure +on: + pull_request: + paths: + - '.github/workflows/**' + - '3rdparty/**' + - '**/*.php' + - '**/lib/**' + - '**/tests/**' + - '**/vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + +concurrency: + group: object-storage-azure-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + azure-primary-tests: + runs-on: ubuntu-latest + needs: changes + + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.3' + coverage: true + + name: php${{ matrix.php-versions }}-azure + + services: + azurite: + image: mcr.microsoft.com/azure-storage/azurite + env: + AZURITE_ACCOUNTS: nextcloud:bmV4dGNsb3Vk + ports: + - 10000:10000 + options: --health-cmd="nc 127.0.0.1 10000 -z" --health-interval=1s --health-retries=30 + + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + steps: + - name: Checkout server + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Nextcloud + env: + OBJECT_STORE: azure + OBJECT_STORE_KEY: nextcloud + OBJECT_STORE_SECRET: bmV4dGNsb3Vk + run: | + composer install + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit + env: + OBJECT_STORE: azure + OBJECT_STORE_KEY: nextcloud + OBJECT_STORE_SECRET: bmV4dGNsb3Vk + run: composer run test -- --group PRIMARY-azure ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-azure + + - name: Azurite logs + if: always() + run: | + docker ps -a + docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done + + azure-primary-summary: + runs-on: ubuntu-latest + needs: [changes, azure-primary-tests] + + if: always() + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.azure-primary-tests.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/object-storage-s3.yml b/.github/workflows/object-storage-s3.yml new file mode 100644 index 0000000000000..e1236a5c39b23 --- /dev/null +++ b/.github/workflows/object-storage-s3.yml @@ -0,0 +1,139 @@ +name: Object storage S3 +on: + pull_request: + paths: + - '.github/workflows/**' + - '3rdparty/**' + - '**/*.php' + - '**/lib/**' + - '**/tests/**' + - '**/vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + +concurrency: + group: object-storage-s3-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + s3-primary-tests-minio: + runs-on: ubuntu-latest + needs: changes + + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.3' + coverage: true + + name: php${{ matrix.php-versions }}-s3 + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + minio: + image: bitnami/minio + env: + MINIO_ROOT_USER: nextcloud + MINIO_ROOT_PASSWORD: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ= + MINIO_DEFAULT_BUCKETS: nextcloud + ports: + - "9000:9000" + + steps: + - name: Checkout server + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Nextcloud + env: + OBJECT_STORE: s3 + OBJECT_STORE_KEY: nextcloud + OBJECT_STORE_SECRET: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ= + run: | + composer install + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: Wait for S3 + run: | + sleep 10 + curl -f -m 1 --retry-connrefused --retry 10 --retry-delay 10 http://localhost:9000/minio/health/ready + + - name: PHPUnit + env: + OBJECT_STORE: s3 + OBJECT_STORE_KEY: nextcloud + OBJECT_STORE_SECRET: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ= + run: composer run test -- --group PRIMARY-s3 ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-s3 + + - name: S3 logs + if: always() + run: | + docker ps -a + docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done + + s3-primary-summary: + runs-on: ubuntu-latest + needs: [changes,s3-primary-tests-minio] + + if: always() + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.s3-primary-tests-minio.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/object-storage-swift.yml b/.github/workflows/object-storage-swift.yml new file mode 100644 index 0000000000000..0179e50d4fb69 --- /dev/null +++ b/.github/workflows/object-storage-swift.yml @@ -0,0 +1,129 @@ +name: Object storage Swift +on: + pull_request: + paths: + - '.github/workflows/**' + - '3rdparty/**' + - '**/*.php' + - '**/lib/**' + - '**/tests/**' + - '**/vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + +concurrency: + group: object-storage-swift-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + swift-primary-tests: + runs-on: ubuntu-latest + needs: changes + + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.3' + coverage: true + + name: php${{ matrix.php-versions }}-swift + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + swift: + image: ghcr.io/cscfi/docker-keystone-swift + ports: + - 5000:5000 + - 8080:8080 + + steps: + - name: Checkout server + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Nextcloud + env: + OBJECT_STORE: swift + OBJECT_STORE_SECRET: veryfast + run: | + composer install + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit + env: + OBJECT_STORE: swift + OBJECT_STORE_SECRET: veryfast + run: composer run test -- --group PRIMARY-swift ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-swift + + - name: Swift logs + if: always() + run: | + docker ps -a + docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done + + swift-primary-summary: + runs-on: ubuntu-latest + needs: [changes,swift-primary-tests] + + if: always() + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.swift-primary-tests.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/s3-primary.yml b/.github/workflows/s3-primary.yml deleted file mode 100644 index 2b7c6e8d7f092..0000000000000 --- a/.github/workflows/s3-primary.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: S3 primary storage -on: - pull_request: - paths: - - '.github/workflows/**' - - '3rdparty/**' - - '**/*.php' - - '**/lib/**' - - '**/tests/**' - - '**/vendor-bin/**' - - '.php-cs-fixer.dist.php' - - 'composer.json' - - 'composer.lock' - -concurrency: - group: s3-primary-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - s3-primary-tests-minio: - runs-on: ubuntu-20.04 - - if: ${{ github.repository_owner != 'nextcloud-gmbh' }} - - strategy: - # do not stop on another job's failure - fail-fast: false - matrix: - php-versions: ['8.0'] - key: ['objectstore', 'objectstore_multibucket'] - - name: php${{ matrix.php-versions }}-${{ matrix.key }}-minio - - services: - minio: - env: - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 - image: bitnami/minio:2021.12.29 - ports: - - "9000:9000" - - steps: - - name: Checkout server - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - with: - submodules: true - - - name: Set up php ${{ matrix.php-versions }} - uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 - with: - php-version: ${{ matrix.php-versions }} - extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Nextcloud - run: | - composer install - mkdir data - echo ' ["class" => "OC\Files\ObjectStore\S3", "arguments" => ["bucket" => "nextcloud", "autocreate" => true, "key" => "minio", "secret" => "minio123", "hostname" => "localhost", "port" => 9000, "use_ssl" => false, "use_path_style" => true, "uploadPartSize" => 52428800]]];' > config/config.php - ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password - php -f index.php - - - name: Wait for S3 - run: | - sleep 10 - curl -f -m 1 --retry-connrefused --retry 10 --retry-delay 10 http://localhost:9000/minio/health/ready - - - name: PHPUnit - run: composer run test:db - - name: S3 logs - if: always() - run: | - docker ps -a - docker logs $(docker ps -aq) - - - s3-primary-summary: - runs-on: ubuntu-latest - needs: [s3-primary-tests-minio] - - if: always() - - steps: - - name: Summary status - run: if ${{ needs.s3-primary-tests-minio.result != 'success' }}; then exit 1; fi diff --git a/tests/lib/Files/ObjectStore/SwiftTest.php b/tests/lib/Files/ObjectStore/SwiftTest.php index 1ea55a846280b..bebfba0c8a4ea 100644 --- a/tests/lib/Files/ObjectStore/SwiftTest.php +++ b/tests/lib/Files/ObjectStore/SwiftTest.php @@ -38,4 +38,8 @@ protected function getInstance() { return new Swift($config['arguments']); } + + public function testFseekSize() { + $this->markTestSkipped('Swift does not support seeking at the moment'); + } } diff --git a/tests/preseed-config.php b/tests/preseed-config.php index 16aea87c8a7a2..c62b447128040 100644 --- a/tests/preseed-config.php +++ b/tests/preseed-config.php @@ -25,71 +25,52 @@ 'arguments' => [ 'bucket' => 'nextcloud', 'autocreate' => true, - 'key' => 'nextcloud', - 'secret' => 'nextcloud', - 'hostname' => getenv('DRONE') === 'true' ? 'minio' : 'localhost', + 'key' => getenv('OBJECT_STORE_KEY') ?: 'nextcloud', + 'secret' => getenv('OBJECT_STORE_SECRET') ?: 'nextcloud', + 'hostname' => getenv('OBJECT_STORE_HOST') ?: 'localhost', 'port' => 9000, 'use_ssl' => false, // required for some non amazon s3 implementations 'use_path_style' => true ] ]; -} -if (getenv('OBJECT_STORE') === 'swift') { - $swiftHost = getenv('DRONE') === 'true' ? 'dockswift' : 'localhost'; - - if (getenv('SWIFT-AUTH') === 'v2.0') { - $CONFIG['objectstore'] = [ - 'class' => 'OC\\Files\\ObjectStore\\Swift', - 'arguments' => [ - 'autocreate' => true, - 'username' => 'swift', - 'tenantName' => 'service', - 'password' => 'swift', - 'serviceName' => 'swift', - 'region' => 'regionOne', - 'url' => "http://$swiftHost:5000/v2.0", - 'bucket' => 'nextcloud' - ] - ]; - } else { - $CONFIG['objectstore'] = [ - 'class' => 'OC\\Files\\ObjectStore\\Swift', - 'arguments' => [ - 'autocreate' => true, - 'user' => [ - 'name' => 'swift', - 'password' => 'swift', - 'domain' => [ - 'name' => 'default', - ] - ], - 'scope' => [ - 'project' => [ - 'name' => 'service', - 'domain' => [ - 'name' => 'default', - ], - ], - ], - 'tenantName' => 'service', - 'serviceName' => 'swift', - 'region' => 'regionOne', - 'url' => "http://$swiftHost:5000/v3", - 'bucket' => 'nextcloud' - ] - ]; - } -} -if (getenv('OBJECT_STORE') === 'azure') { +} elseif (getenv('OBJECT_STORE') === 'azure') { $CONFIG['objectstore'] = [ 'class' => 'OC\\Files\\ObjectStore\\Azure', 'arguments' => [ 'container' => 'test', - 'account_name' => 'devstoreaccount1', - 'account_key' => 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', - 'endpoint' => 'http://' . (getenv('DRONE') === 'true' ? 'azurite' : 'localhost') . ':10000/devstoreaccount1', + 'account_name' => getenv('OBJECT_STORE_KEY') ?: 'devstoreaccount1', + 'account_key' => getenv('OBJECT_STORE_SECRET') ?: 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', + 'endpoint' => 'http://' . (getenv('OBJECT_STORE_HOST') ?: 'localhost') . ':10000/' . (getenv('OBJECT_STORE_KEY') ?: 'devstoreaccount1'), 'autocreate' => true ] ]; +} elseif (getenv('OBJECT_STORE') === 'swift') { + $swiftHost = getenv('OBJECT_STORE_HOST') ?: 'localhost:5000'; + + $CONFIG['objectstore'] = [ + 'class' => 'OC\\Files\\ObjectStore\\Swift', + 'arguments' => [ + 'autocreate' => true, + 'user' => [ + 'name' => getenv('OBJECT_STORE_KEY') ?: 'swift', + 'password' => getenv('OBJECT_STORE_SECRET') ?: 'swift', + 'domain' => [ + 'name' => 'Default', + ], + ], + 'scope' => [ + 'project' => [ + 'name' => 'service', + 'domain' => [ + 'name' => 'Default', + ], + ], + ], + 'serviceName' => 'service', + 'region' => 'RegionOne', + 'url' => "http://$swiftHost/v3", + 'bucket' => 'nextcloud', + ] + ]; } From fd378c1b12b074dd86bd25d2a7c1c7c5ba36d2e6 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sun, 29 Oct 2023 02:26:43 +0200 Subject: [PATCH 11/17] chore(tests): Migrate memcached unit test from drone to GitHub Action Signed-off-by: Ferdinand Thiessen --- .drone.yml | 26 ----- .github/workflows/phpunit-memcached.yml | 124 ++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/phpunit-memcached.yml diff --git a/.drone.yml b/.drone.yml index 4cc000c628341..ecbdedbe80b57 100644 --- a/.drone.yml +++ b/.drone.yml @@ -217,31 +217,5 @@ trigger: - push --- -kind: pipeline -name: memcache-memcached - -steps: -- name: submodules - image: ghcr.io/nextcloud/continuous-integration-alpine-git:latest - commands: - - git submodule update --init -- name: memcache-memcached - image: ghcr.io/nextcloud/continuous-integration-php8.0-memcached:latest - commands: - - phpenmod xdebug - - service memcached restart - - ./autotest.sh sqlite tests/lib/Memcache/MemcachedTest.php - - wget https://codecov.io/bash -O codecov.sh - - sh -c "if [ '$DRONE_BUILD_EVENT' = 'pull_request' ]; then bash codecov.sh -B $DRONE_BRANCH -C $DRONE_COMMIT -P $DRONE_PULL_REQUEST -t 117641e2-a9e8-4b7b-984b-ae872d9b05f5 -f tests/autotest-clover-sqlite.xml; fi" - - sh -c "if [ '$DRONE_BUILD_EVENT' != 'pull_request' ]; then bash codecov.sh -B $DRONE_BRANCH -C $DRONE_COMMIT -t 117641e2-a9e8-4b7b-984b-ae872d9b05f5 -f tests/autotest-clover-sqlite.xml; fi" - -trigger: - branch: - - master - - stable* - event: - - pull_request - - push ---- kind: signature hmac: 893f108e2269f7da269bf70b284e40d13932bd2697d0393686c42b287da5afcf diff --git a/.github/workflows/phpunit-memcached.yml b/.github/workflows/phpunit-memcached.yml new file mode 100644 index 0000000000000..e155e9926b631 --- /dev/null +++ b/.github/workflows/phpunit-memcached.yml @@ -0,0 +1,124 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization + +name: PHPUnit memcached + +on: + pull_request: + push: + branches: + - stable* + +permissions: + contents: read + +concurrency: + group: phpunit-memcached-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-memcached: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.2' + coverage: true + + name: Memcached (PHP ${{ matrix.php-versions }}) + + services: + memcached: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 11212:11212/tcp + - 11212:11212/udp + + steps: + - name: Checkout server + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@7fdd3ece872ec7ec4c098ae5ab7637d5e0a96067 # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, memcached, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Set up Nextcloud + run: | + mkdir data + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit memcached tests + run: composer run test -- --group Memcache,Memcached ${{ matrix.coverage && '--coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-memcached + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest + needs: [changes, phpunit-memcached] + + if: always() + + name: phpunit-memcached-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-memcached.result != 'success' }}; then exit 1; fi From 4a1ff0b6da826d16204fb2ea28302c2c6f7d4309 Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Wed, 8 Nov 2023 02:30:33 +0100 Subject: [PATCH 12/17] fix: Run tests only on pull requests and scheduled Signed-off-by: Ferdinand Thiessen --- .github/workflows/object-storage-azure.yml | 12 ++---------- .github/workflows/object-storage-s3.yml | 12 ++---------- .github/workflows/object-storage-swift.yml | 12 ++---------- .github/workflows/phpunit-mariadb.yml | 7 ++----- .github/workflows/phpunit-memcached.yml | 5 ++--- .github/workflows/phpunit-mysql.yml | 9 +++------ .github/workflows/phpunit-nodb.yml | 9 +++------ .github/workflows/phpunit-oci.yml | 7 +++++-- .github/workflows/phpunit-pgsql.yml | 9 +++------ .github/workflows/phpunit-sqlite.yml | 9 +++------ 10 files changed, 27 insertions(+), 64 deletions(-) diff --git a/.github/workflows/object-storage-azure.yml b/.github/workflows/object-storage-azure.yml index 72389c5a816a2..ef0e29b117bde 100644 --- a/.github/workflows/object-storage-azure.yml +++ b/.github/workflows/object-storage-azure.yml @@ -1,16 +1,8 @@ name: Object storage azure on: pull_request: - paths: - - '.github/workflows/**' - - '3rdparty/**' - - '**/*.php' - - '**/lib/**' - - '**/tests/**' - - '**/vendor-bin/**' - - '.php-cs-fixer.dist.php' - - 'composer.json' - - 'composer.lock' + schedule: + - cron: "15 2 * * *" concurrency: group: object-storage-azure-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/object-storage-s3.yml b/.github/workflows/object-storage-s3.yml index e1236a5c39b23..8482fc5cc88d4 100644 --- a/.github/workflows/object-storage-s3.yml +++ b/.github/workflows/object-storage-s3.yml @@ -1,16 +1,8 @@ name: Object storage S3 on: pull_request: - paths: - - '.github/workflows/**' - - '3rdparty/**' - - '**/*.php' - - '**/lib/**' - - '**/tests/**' - - '**/vendor-bin/**' - - '.php-cs-fixer.dist.php' - - 'composer.json' - - 'composer.lock' + schedule: + - cron: "15 2 * * *" concurrency: group: object-storage-s3-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/object-storage-swift.yml b/.github/workflows/object-storage-swift.yml index 0179e50d4fb69..d6ef067d2d075 100644 --- a/.github/workflows/object-storage-swift.yml +++ b/.github/workflows/object-storage-swift.yml @@ -1,16 +1,8 @@ name: Object storage Swift on: pull_request: - paths: - - '.github/workflows/**' - - '3rdparty/**' - - '**/*.php' - - '**/lib/**' - - '**/tests/**' - - '**/vendor-bin/**' - - '.php-cs-fixer.dist.php' - - 'composer.json' - - 'composer.lock' + schedule: + - cron: "15 2 * * *" concurrency: group: object-storage-swift-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/phpunit-mariadb.yml b/.github/workflows/phpunit-mariadb.yml index cf213d1c1b457..b3ebdfe16e6cb 100644 --- a/.github/workflows/phpunit-mariadb.yml +++ b/.github/workflows/phpunit-mariadb.yml @@ -7,11 +7,8 @@ name: PHPUnit mariadb on: pull_request: - push: - branches: - - main - - master - - stable* + schedule: + - cron: "5 2 * * *" permissions: contents: read diff --git a/.github/workflows/phpunit-memcached.yml b/.github/workflows/phpunit-memcached.yml index e155e9926b631..9ec788b31eb6d 100644 --- a/.github/workflows/phpunit-memcached.yml +++ b/.github/workflows/phpunit-memcached.yml @@ -7,9 +7,8 @@ name: PHPUnit memcached on: pull_request: - push: - branches: - - stable* + schedule: + - cron: "5 2 * * *" permissions: contents: read diff --git a/.github/workflows/phpunit-mysql.yml b/.github/workflows/phpunit-mysql.yml index 1c47951360341..1bf33ca51f753 100644 --- a/.github/workflows/phpunit-mysql.yml +++ b/.github/workflows/phpunit-mysql.yml @@ -7,11 +7,8 @@ name: PHPUnit mysql on: pull_request: - push: - branches: - - main - - master - - stable* + schedule: + - cron: "5 2 * * *" permissions: contents: read @@ -25,7 +22,7 @@ jobs: runs-on: ubuntu-latest outputs: - src: ${{ steps.changes.outputs.src}} + src: ${{ steps.changes.outputs.src }} steps: - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 diff --git a/.github/workflows/phpunit-nodb.yml b/.github/workflows/phpunit-nodb.yml index b9dea3c28ee58..2454da43b07d1 100644 --- a/.github/workflows/phpunit-nodb.yml +++ b/.github/workflows/phpunit-nodb.yml @@ -8,11 +8,8 @@ name: PHPUnit nodb on: pull_request: - push: - branches: - - main - - master - - stable* + schedule: + - cron: "5 2 * * *" permissions: contents: read @@ -26,7 +23,7 @@ jobs: runs-on: ubuntu-latest outputs: - src: ${{ steps.changes.outputs.src}} + src: ${{ steps.changes.outputs.src }} steps: - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 diff --git a/.github/workflows/phpunit-oci.yml b/.github/workflows/phpunit-oci.yml index fb2081876a75d..e24a0dd899059 100644 --- a/.github/workflows/phpunit-oci.yml +++ b/.github/workflows/phpunit-oci.yml @@ -1,6 +1,9 @@ name: PHPUnit oci -on: pull_request +on: + pull_request: + schedule: + - cron: "5 2 * * *" permissions: contents: read @@ -14,7 +17,7 @@ jobs: runs-on: ubuntu-latest outputs: - src: ${{ steps.changes.outputs.src}} + src: ${{ steps.changes.outputs.src }} steps: - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 diff --git a/.github/workflows/phpunit-pgsql.yml b/.github/workflows/phpunit-pgsql.yml index ddcd20ed51ccd..94d5562cd6e33 100644 --- a/.github/workflows/phpunit-pgsql.yml +++ b/.github/workflows/phpunit-pgsql.yml @@ -7,11 +7,8 @@ name: PHPUnit pgsql on: pull_request: - push: - branches: - - main - - master - - stable* + schedule: + - cron: "5 2 * * *" permissions: contents: read @@ -25,7 +22,7 @@ jobs: runs-on: ubuntu-latest outputs: - src: ${{ steps.changes.outputs.src}} + src: ${{ steps.changes.outputs.src }} steps: - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 diff --git a/.github/workflows/phpunit-sqlite.yml b/.github/workflows/phpunit-sqlite.yml index 61710f31b6fcd..fa9728c2a5355 100644 --- a/.github/workflows/phpunit-sqlite.yml +++ b/.github/workflows/phpunit-sqlite.yml @@ -7,11 +7,8 @@ name: PHPUnit sqlite on: pull_request: - push: - branches: - - main - - master - - stable* + schedule: + - cron: "5 2 * * *" permissions: contents: read @@ -25,7 +22,7 @@ jobs: runs-on: ubuntu-latest outputs: - src: ${{ steps.changes.outputs.src}} + src: ${{ steps.changes.outputs.src }} steps: - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 From 4465caafa2d8ff98ed8c0a2b2628e3291942988d Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Wed, 8 Nov 2023 02:32:17 +0100 Subject: [PATCH 13/17] chore: Migrate PHPUnit files_external S3 test Signed-off-by: Ferdinand Thiessen --- .github/workflows/files-external-s3.yml | 186 ++++++++++++++++++++++++ .github/workflows/s3-external.yml | 130 ----------------- 2 files changed, 186 insertions(+), 130 deletions(-) create mode 100644 .github/workflows/files-external-s3.yml delete mode 100644 .github/workflows/s3-external.yml diff --git a/.github/workflows/files-external-s3.yml b/.github/workflows/files-external-s3.yml new file mode 100644 index 0000000000000..a46ab25591eb3 --- /dev/null +++ b/.github/workflows/files-external-s3.yml @@ -0,0 +1,186 @@ +name: PHPUnit files_external S3 +on: + pull_request: + schedule: + - cron: "5 2 * * *" + +concurrency: + group: files-external-s3-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - 'apps/files_external/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + files-external-s3-minio: + runs-on: ubuntu-latest + needs: changes + + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.3' + coverage: true + + name: php${{ matrix.php-versions }}-s3 + + services: + minio: + image: bitnami/minio + env: + MINIO_ROOT_USER: nextcloud + MINIO_ROOT_PASSWORD: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ= + MINIO_DEFAULT_BUCKETS: nextcloud + ports: + - "9000:9000" + + steps: + - name: Checkout server + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Nextcloud + env: + OBJECT_STORE_KEY: nextcloud + OBJECT_STORE_SECRET: bWluaW8tc2VjcmV0LWtleS1uZXh0Y2xvdWQ= + run: | + composer install + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable --force files_external + echo " true, 'secret' => 'actually-not-secret', 'passwordsalt' => 'actually-not-secret', 'hostname' => 'localhost','key' => '$OBJECT_STORE_KEY','secret' => '$OBJECT_STORE_SECRET', 'bucket' => 'bucket', 'port' => 9000, 'use_ssl' => false, 'autocreate' => true, 'use_path_style' => true];" > apps/files_external/tests/config.amazons3.php + + - name: Wait for S3 + run: | + sleep 10 + curl -f -m 1 --retry-connrefused --retry 10 --retry-delay 10 http://localhost:9000/minio/health/ready + + - name: PHPUnit + run: composer run test:files_external -- \ + apps/files_external/tests/Storage/Amazons3Test.php \ + apps/files_external/tests/Storage/VersionedAmazonS3Test.php \ + ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-files-external-s3 + + - name: S3 logs + if: always() + run: | + docker ps -a + docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done + + files-external-s3-localstack: + runs-on: ubuntu-latest + needs: changes + + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} + + strategy: + matrix: + php-versions: ['8.0', '8.1', '8.2', '8.3'] + include: + - php-versions: '8.3' + coverage: true + + name: php${{ matrix.php-versions }}-s3 + + services: + localstack: + env: + SERVICES: s3 + DEBUG: 1 + image: localstack/localstack + ports: + - "4566:4566" + + steps: + - name: Checkout server + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@4bd44f22a98a19e0950cbad5f31095157cc9621b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Nextcloud + run: | + composer install + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable --force files_external + echo " true,'hostname' => 'localhost','key' => 'ignored','secret' => 'ignored', 'bucket' => 'bucket', 'port' => 4566, 'use_ssl' => false, 'autocreate' => true, 'use_path_style' => true];" > apps/files_external/tests/config.amazons3.php + + - name: PHPUnit + run: composer run test:files_external -- \ + apps/files_external/tests/Storage/Amazons3Test.php \ + apps/files_external/tests/Storage/VersionedAmazonS3Test.php \ + ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-files-external-s3 + + - name: S3 logs + if: always() + run: | + docker ps -a + docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done + + s3-external-summary: + runs-on: ubuntu-latest + needs: [changes, files-external-s3-minio, files-external-s3-localstack] + + if: always() + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.files-external-s3-minio.result != 'success' && needs.files-external-s3-localstack.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/s3-external.yml b/.github/workflows/s3-external.yml deleted file mode 100644 index ae859a97df3d1..0000000000000 --- a/.github/workflows/s3-external.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: S3 External storage -on: - pull_request: - paths: - - 'apps/files_external/**' - -concurrency: - group: s3-external-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - APP_NAME: files_external - -jobs: - s3-external-tests-minio: - runs-on: ubuntu-latest - - if: ${{ github.repository_owner != 'nextcloud-gmbh' }} - - strategy: - # do not stop on another job's failure - fail-fast: false - matrix: - php-versions: ['8.0', '8.1'] - - name: php${{ matrix.php-versions }}-minio - - services: - minio: - env: - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 - image: bitnami/minio:2021.10.6 - ports: - - "9000:9000" - - steps: - - name: Checkout server - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - with: - submodules: true - - - name: Set up php ${{ matrix.php-versions }} - uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 - with: - php-version: ${{ matrix.php-versions }} - extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Nextcloud - run: | - composer install - mkdir data - ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password - ./occ app:enable --force ${{ env.APP_NAME }} - php -S localhost:8080 & - - name: PHPUnit - run: | - echo " true, 'secret' => 'actually-not-secret', 'passwordsalt' => 'actually-not-secret', 'hostname' => 'localhost','key' => 'minio','secret' => 'minio123', 'bucket' => 'bucket', 'port' => 9000, 'use_ssl' => false, 'autocreate' => true, 'use_path_style' => true];" > apps/${{ env.APP_NAME }}/tests/config.amazons3.php - composer run test:files_external apps/files_external/tests/Storage/Amazons3Test.php - composer run test:files_external apps/files_external/tests/Storage/VersionedAmazonS3Test.php - - name: S3 logs - if: always() - run: | - docker ps -a - docker logs $(docker ps -aq) - s3-external-tests-localstack: - runs-on: ubuntu-latest - - if: ${{ github.repository_owner != 'nextcloud-gmbh' }} - - strategy: - # do not stop on another job's failure - fail-fast: false - matrix: - php-versions: ['8.0', '8.1'] - - name: php${{ matrix.php-versions }}-localstack - - services: - minio: - env: - SERVICES: s3 - DEBUG: 1 - image: localstack/localstack:0.12.7 - ports: - - "4566:4566" - - steps: - - name: Checkout server - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - with: - submodules: true - - - name: Set up php ${{ matrix.php-versions }} - uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 - with: - php-version: ${{ matrix.php-versions }} - extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Nextcloud - run: | - composer install - mkdir data - ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password - ./occ app:enable --force ${{ env.APP_NAME }} - php -S localhost:8080 & - - name: PHPUnit - run: | - echo " true,'hostname' => 'localhost','key' => 'ignored','secret' => 'ignored', 'bucket' => 'bucket', 'port' => 4566, 'use_ssl' => false, 'autocreate' => true, 'use_path_style' => true];" > apps/${{ env.APP_NAME }}/tests/config.amazons3.php - composer run test:files_external apps/files_external/tests/Storage/Amazons3Test.php - composer run test:files_external apps/files_external/tests/Storage/VersionedAmazonS3Test.php - - name: S3 logs - if: always() - run: | - docker ps -a - docker logs $(docker ps -aq) - - s3-external-summary: - runs-on: ubuntu-latest - needs: [s3-external-tests-minio, s3-external-tests-localstack] - - if: always() - - steps: - - name: Summary status - run: if ${{ needs.s3-external-tests-minio.result != 'success' }} || ${{ needs.s3-external-tests-localstack.result != 'success' }}; then exit 1; fi From 590b4fdff7635f7474ef2ff37ea8385d091aa52c Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 26 Jan 2024 14:56:16 +0100 Subject: [PATCH 14/17] fix(CI): Sign .drone.yml Signed-off-by: Joas Schilling --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index ecbdedbe80b57..1e0a0c55e87d0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -218,4 +218,4 @@ trigger: --- kind: signature -hmac: 893f108e2269f7da269bf70b284e40d13932bd2697d0393686c42b287da5afcf +hmac: c0e95a282303de842a645a780b391964eb839e4cb11446e4cc9a19ef1a661383 From 6dffc9bc17bd494de4e9b6c1d51d9707c2a5b0b2 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 26 Jan 2024 16:05:09 +0100 Subject: [PATCH 15/17] fix(CS): Fix coding style Signed-off-by: Joas Schilling --- apps/encryption/tests/Crypto/EncryptAllTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/encryption/tests/Crypto/EncryptAllTest.php b/apps/encryption/tests/Crypto/EncryptAllTest.php index 7c7e6a7f7ab4e..78b9b5f7e3718 100644 --- a/apps/encryption/tests/Crypto/EncryptAllTest.php +++ b/apps/encryption/tests/Crypto/EncryptAllTest.php @@ -125,7 +125,7 @@ protected function setUp(): void { /** * We need format method to return a string * @var OutputFormatterInterface|\PHPUnit\Framework\MockObject\MockObject - */ + */ $outputFormatter = $this->createMock(OutputFormatterInterface::class); $outputFormatter->method('isDecorated')->willReturn(false); $outputFormatter->method('format')->willReturnArgument(0); From d086b6c81d32c83a0bd480cc29819a4d5e1cc425 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 26 Jan 2024 16:16:37 +0100 Subject: [PATCH 16/17] fix(tests): Make PHP 8.1.2 with Datetime Diff Bug pass the notifier test Signed-off-by: Joas Schilling --- .../tests/unit/CalDAV/Reminder/NotifierTest.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/dav/tests/unit/CalDAV/Reminder/NotifierTest.php b/apps/dav/tests/unit/CalDAV/Reminder/NotifierTest.php index 08677d202670b..10c421921b03c 100644 --- a/apps/dav/tests/unit/CalDAV/Reminder/NotifierTest.php +++ b/apps/dav/tests/unit/CalDAV/Reminder/NotifierTest.php @@ -142,6 +142,14 @@ public function testPrepareWrongSubject(): void { $this->notifier->prepare($notification, 'en'); } + private static function hasPhpDatetimeDiffBug(): bool { + $d1 = \DateTime::createFromFormat(\DateTimeInterface::ATOM, '2023-11-22T11:52:00+01:00'); + $d2 = new \DateTime('2023-11-22T10:52:03', new \DateTimeZone('UTC')); + + // The difference is 3 seconds, not -1year+11months+… + return $d1->diff($d2)->y < 0; + } + public function dataPrepare(): array { return [ [ @@ -150,7 +158,7 @@ public function dataPrepare(): array { 'title' => 'Title of this event', 'start_atom' => '2005-08-15T15:52:01+02:00' ], - 'Title of this event (in 1 hour, 52 minutes)', + self::hasPhpDatetimeDiffBug() ? 'Title of this event' : 'Title of this event (in 1 hour, 52 minutes)', [ 'title' => 'Title of this event', 'description' => null, @@ -172,7 +180,7 @@ public function dataPrepare(): array { 'title' => 'Title of this event', 'start_atom' => '2005-08-15T13:00:00+02:00', ], - 'Title of this event (1 hour ago)', + self::hasPhpDatetimeDiffBug() ? 'Title of this event' : 'Title of this event (1 hour ago)', [ 'title' => 'Title of this event', 'description' => null, @@ -281,7 +289,7 @@ public function testPassedEvent(): void { $notification->expects($this->once()) ->method('setParsedSubject') - ->with('Title of this event (6 hours ago)') + ->with(self::hasPhpDatetimeDiffBug() ? 'Title of this event' : 'Title of this event (6 hours ago)') ->willReturnSelf(); $this->expectException(AlreadyProcessedException::class); From 58506952840049b69224ae119f89da0f25737a11 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 26 Jan 2024 20:38:43 +0100 Subject: [PATCH 17/17] fix(tests): Add messages to assertions to explain what failed Signed-off-by: Joas Schilling --- tests/lib/Files/ObjectStore/S3Test.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php index 33130a152a1e1..c8333ca1ea38d 100644 --- a/tests/lib/Files/ObjectStore/S3Test.php +++ b/tests/lib/Files/ObjectStore/S3Test.php @@ -113,7 +113,7 @@ public function assertNoUpload($objectUrn) { 'Bucket' => $s3->getBucket(), 'Prefix' => $objectUrn, ]); - $this->assertArrayNotHasKey('Uploads', $uploads); + $this->assertArrayNotHasKey('Uploads', $uploads, 'Assert is not uploaded'); } public function testEmptyUpload() { @@ -125,11 +125,11 @@ public function testEmptyUpload() { $s3->writeObject('emptystream', $emptyStream); $this->assertNoUpload('emptystream'); - $this->assertTrue($s3->objectExists('emptystream')); + $this->assertTrue($s3->objectExists('emptystream'), 'Object exists on S3'); $thrown = false; try { - self::assertFalse($s3->readObject('emptystream')); + self::assertFalse($s3->readObject('emptystream'), 'Reading empty stream object should return false'); } catch (\Exception $e) { // An exception is expected here since 0 byte files are wrapped // to be read from an empty memory stream in the ObjectStoreStorage @@ -164,20 +164,20 @@ public function testFileSizes($size) { $s3->writeObject('testfilesizes', $sourceStream); $this->assertNoUpload('testfilesizes'); - self::assertTrue($s3->objectExists('testfilesizes')); + self::assertTrue($s3->objectExists('testfilesizes'), 'Object exists on S3'); $result = $s3->readObject('testfilesizes'); // compare first 100 bytes - self::assertEquals(str_repeat('A', 100), fread($result, 100)); + self::assertEquals(str_repeat('A', 100), fread($result, 100), 'Compare first 100 bytes'); - // compare 100 bytes + // compare last 100 bytes fseek($result, $size - 100); - self::assertEquals(str_repeat('A', 100), fread($result, 100)); + self::assertEquals(str_repeat('A', 100), fread($result, 100), 'Compare last 100 bytes'); // end of file reached fseek($result, $size); - self::assertTrue(feof($result)); + self::assertTrue(feof($result), 'End of file reached'); $this->assertNoUpload('testfilesizes'); }