From 274ccd97294d4a7bf950f8a835c3c079b18fa5bd Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Dec 2024 06:18:16 +0100 Subject: [PATCH 1/5] test(federation): Run cross major federation on CI Signed-off-by: Joas Schilling --- .github/workflows/integration-federation.yml | 275 +++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 .github/workflows/integration-federation.yml diff --git a/.github/workflows/integration-federation.yml b/.github/workflows/integration-federation.yml new file mode 100644 index 00000000000..f36823c0ea3 --- /dev/null +++ b/.github/workflows/integration-federation.yml @@ -0,0 +1,275 @@ +# SPDX-FileCopyrightText: 2023-2024 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: MIT + +name: Integration Federation + +on: + pull_request: + schedule: + - cron: "5 2 * * *" + +permissions: + contents: read + +concurrency: + group: integration-federation-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest-low + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - 'appinfo/**' + - 'lib/**' + - 'templates/**' + - 'tests/integration/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + + integration-federation: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + fail-fast: false + matrix: + include: + - test-suite: 'federation' + php-versions: '8.2' + host-server-versions: 'master' + host-spreed-versions: ${{ github.event.pull_request.head.sha }} + host-guests-versions: 'master' + host-circles-versions: 'master' + host-call-summary-bot-versions: 'main' + host-notifications-versions: 'master' + remote-server-versions: 'stable30' + remote-spreed-versions: 'stable30' + remote-guests-versions: 'stable30' + remote-circles-versions: 'stable30' + remote-call-summary-bot-versions: 'main' + remote-notifications-versions: 'stable30' + + services: + oracle: + image: ghcr.io/gvenzl/oracle-xe:11 + + # Provide passwords and other environment variables to container + env: + ORACLE_RANDOM_PASSWORD: true + APP_USER: autotest + APP_USER_PASSWORD: owncloud + + # Forward Oracle port + ports: + - 1521:1521/tcp + + # Provide healthcheck script options for startup + options: >- + --health-cmd healthcheck.sh + --health-interval 10s + --health-timeout 5s + --health-retries 10 + + mysql: + image: ghcr.io/nextcloud/continuous-integration-mysql-8.4:latest + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + + steps: + - name: Set app env + run: | + # Split and keep last + echo "APP_NAME=${GITHUB_REPOSITORY##*/}" >> $GITHUB_ENV + + - name: Checkout server - Host + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + path: host + submodules: true + repository: nextcloud/server + ref: ${{ matrix.host-server-versions }} + + - name: Checkout spreed app - Host + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + path: host/apps/spreed + ref: ${{ matrix.host-spreed-versions }} + + - name: Checkout call_summary_bot app - Host + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/call_summary_bot + path: host/apps/call_summary_bot + ref: ${{ matrix.host-call-summary-bot-versions }} + + - name: Checkout circles app - Host + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/circles + path: host/apps/circles + ref: ${{ matrix.host-circles-versions }} + + - name: Checkout guests app - Host + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/guests + path: host/apps/guests + ref: ${{ matrix.host-guests-versions }} + + - name: Checkout notifications app - Host + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/notifications + path: host/apps/notifications + ref: ${{ matrix.host-notifications-versions }} + + - name: Checkout server - Remote + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + path: remote + submodules: true + repository: nextcloud/server + ref: ${{ matrix.remote-server-versions }} + + - name: Checkout spreed app - Remote + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + path: remote/apps/spreed + ref: ${{ matrix.remote-spreed-versions }} + + - name: Checkout call_summary_bot app - Remote + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/call_summary_bot + path: remote/apps/call_summary_bot + ref: ${{ matrix.remote-call-summary-bot-versions }} + + - name: Checkout circles app - Remote + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/circles + path: remote/apps/circles + ref: ${{ matrix.remote-circles-versions }} + + - name: Checkout guests app - Remote + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/guests + path: remote/apps/guests + ref: ${{ matrix.remote-guests-versions }} + + - name: Checkout notifications app - Remote + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + repository: nextcloud/notifications + path: remote/apps/notifications + ref: ${{ matrix.remote-notifications-versions }} + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@2e947f1f6932d141d076ca441d0e1e881775e95b # v2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: apcu, bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql, oci8, sqlite, pdo_sqlite + coverage: none + ini-file: development + # Temporary workaround for missing pcntl_* in PHP 8.3: ini-values: apc.enable_cli=on + ini-values: apc.enable_cli=on, disable_functions= + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up production dependencies - Host + working-directory: host/apps/${{ env.APP_NAME }} + run: composer i --no-dev + + - name: Set up production dependencies - Remote + working-directory: remote/apps/${{ env.APP_NAME }} + run: composer i --no-dev + + - name: Set up behat dependencies + working-directory: host/apps/${{ env.APP_NAME }}/tests/integration + run: composer i + + - name: Set up Nextcloud - Host + env: + DB_PORT: 1521 + working-directory: host/ + run: | + mkdir data + ./occ maintenance:install --verbose --database=oci --database-name=XE --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=autotest --database-pass=owncloud --admin-user admin --admin-pass admin + ./occ config:system:set debug --value=true --type=boolean + ./occ config:system:set hashing_default_password --value=true --type=boolean + ./occ config:system:set memcache.local --value="\\OC\\Memcache\\APCu" + ./occ config:system:set memcache.distributed --value="\\OC\\Memcache\\APCu" + ./occ app:enable --force ${{ env.APP_NAME }} + ./occ app:enable --force call_summary_bot + ./occ app:enable --force circles + ./occ app:enable --force guests + ./occ app:enable --force notifications + + - name: Set up Nextcloud - Remote + env: + DB_PORT: 4444 + working-directory: remote/ + run: | + mkdir data + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + ./occ config:system:set debug --value=true --type=boolean + ./occ config:system:set hashing_default_password --value=true --type=boolean + ./occ config:system:set memcache.local --value="\\OC\\Memcache\\APCu" + ./occ config:system:set memcache.distributed --value="\\OC\\Memcache\\APCu" + ./occ app:enable --force ${{ env.APP_NAME }} + ./occ app:enable --force call_summary_bot + ./occ app:enable --force circles + ./occ app:enable --force guests + ./occ app:enable --force notifications + echo 'REMOTE_ROOT_DIR='$PWD >> $GITHUB_ENV + + - name: Run integration + working-directory: host/apps/${{ env.APP_NAME }}/tests/integration + run: REMOTE_ROOT_DIR=${{ env.REMOTE_ROOT_DIR }} bash run.sh features/${{ matrix.test-suite }} + + - name: Print logs - Host + if: always() + working-directory: host/ + run: | + cat data/nextcloud.log + + - name: Print logs - Remote + if: always() + working-directory: remote/ + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest-low + needs: [changes, integration-federation] + + if: always() + + name: integration-federation-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.integration-federation.result != 'success' }}; then exit 1; fi From eb2e3e7ffaf5bb14b573af1f375f0c47a6703637 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Dec 2024 08:46:55 +0100 Subject: [PATCH 2/5] test: Allow 2 different Nextcloud code paths for testing federation Signed-off-by: Joas Schilling --- .../features/bootstrap/CommandLineTrait.php | 8 +- .../features/bootstrap/FederationContext.php | 2 +- tests/integration/run.sh | 159 ++++++++++-------- 3 files changed, 96 insertions(+), 73 deletions(-) diff --git a/tests/integration/features/bootstrap/CommandLineTrait.php b/tests/integration/features/bootstrap/CommandLineTrait.php index d6098369afc..34c9827ce7f 100644 --- a/tests/integration/features/bootstrap/CommandLineTrait.php +++ b/tests/integration/features/bootstrap/CommandLineTrait.php @@ -51,9 +51,11 @@ public function runOcc($args = [], $env = []) { $argString = implode(' ', $args); if ($this->currentServer === 'REMOTE') { - $env['NEXTCLOUD_CONFIG_DIR'] = getenv('REAL_FEDERATED_SERVER_CONFIG_DIR'); + $env['NEXTCLOUD_CONFIG_DIR'] = getenv('NEXTCLOUD_REMOTE_CONFIG_DIR'); + $serverRootDir = getenv('NEXTCLOUD_REMOTE_ROOT_DIR'); } else { - $env['NEXTCLOUD_CONFIG_DIR'] = getenv('MAIN_SERVER_CONFIG_DIR'); + $env['NEXTCLOUD_CONFIG_DIR'] = getenv('NEXTCLOUD_HOST_CONFIG_DIR'); + $serverRootDir = getenv('NEXTCLOUD_HOST_ROOT_DIR'); } $descriptor = [ @@ -61,7 +63,7 @@ public function runOcc($args = [], $env = []) { 1 => ['pipe', 'w'], 2 => ['pipe', 'w'], ]; - $process = proc_open('php console.php ' . $argString, $descriptor, $pipes, $this->ocPath, $env); + $process = proc_open('php ' . $serverRootDir . '/console.php ' . $argString, $descriptor, $pipes, $this->ocPath, $env); $this->lastStdOut = stream_get_contents($pipes[1]); $this->lastStdErr = stream_get_contents($pipes[2]); $this->lastCode = proc_close($process); diff --git a/tests/integration/features/bootstrap/FederationContext.php b/tests/integration/features/bootstrap/FederationContext.php index 2f96af13b27..3ed00c452cf 100644 --- a/tests/integration/features/bootstrap/FederationContext.php +++ b/tests/integration/features/bootstrap/FederationContext.php @@ -27,7 +27,7 @@ public function startFederatedServer() { } $port = getenv('PORT_FED'); - $rootDir = getenv('NEXTCLOUD_ROOT_DIR'); + $rootDir = getenv('NEXTCLOUD_HOST_ROOT_DIR'); self::$phpFederatedServerPid = exec('php -S localhost:' . $port . ' -t ' . $rootDir . ' >/dev/null & echo $!'); } diff --git a/tests/integration/run.sh b/tests/integration/run.sh index 5ad3cd73087..70189e85c2e 100755 --- a/tests/integration/run.sh +++ b/tests/integration/run.sh @@ -43,29 +43,49 @@ echo -e "Running on process ID: \033[1;35m$PHPPID2\033[0m" # Output filtered federated php server logs tail -f phpserver_fed.log | grep --line-buffered -v -E ":[0-9]+ Accepted$" | grep --line-buffered -v -E ":[0-9]+ Closing$" & +occ_host() { + NEXTCLOUD_CONFIG_DIR=${MAIN_SERVER_CONFIG_DIR} ${ROOT_DIR}/occ "$@" +} + +occ_remote() { + NEXTCLOUD_CONFIG_DIR=${REAL_FEDERATED_SERVER_CONFIG_DIR} ${REMOTE_ROOT_DIR}/occ "$@" +} + MAIN_SERVER_CONFIG_DIR=${ROOT_DIR}/config -MAIN_SERVER_DATA_DIR=$(${ROOT_DIR}/occ config:system:get datadirectory) -MAIN_SERVER_APPS_PATHS=$(${ROOT_DIR}/occ config:system:get apps_paths --output json) -REAL_FEDERATED_SERVER_CONFIG_DIR="$MAIN_SERVER_DATA_DIR/tests-talk-real-federated-server/config" -REAL_FEDERATED_SERVER_DATA_DIR="$MAIN_SERVER_DATA_DIR/tests-talk-real-federated-server/data" -DESTROY_REAL_FEDERATED_SERVER=false - -if [ ! -d "$REAL_FEDERATED_SERVER_CONFIG_DIR" ] || NEXTCLOUD_CONFIG_DIR="$REAL_FEDERATED_SERVER_CONFIG_DIR" ${ROOT_DIR}/occ status | grep "installed: false"; then - DESTROY_REAL_FEDERATED_SERVER=true - if [ $CI ]; then - DESTROY_REAL_FEDERATED_SERVER=false - fi - echo '' - echo -e "\033[0;31mReal federated server not installed in $REAL_FEDERATED_SERVER_CONFIG_DIR\033[0m" - echo -e "\033[0;33mPerforming basic SQLite installation with data directory in $REAL_FEDERATED_SERVER_DATA_DIR\033[0m" - mkdir --parents "$REAL_FEDERATED_SERVER_CONFIG_DIR" "$REAL_FEDERATED_SERVER_DATA_DIR" - NEXTCLOUD_CONFIG_DIR="$REAL_FEDERATED_SERVER_CONFIG_DIR" ${ROOT_DIR}/occ maintenance:install --admin-pass=admin --data-dir="$REAL_FEDERATED_SERVER_DATA_DIR" +MAIN_SERVER_DATA_DIR=$(occ_host config:system:get datadirectory) +MAIN_SERVER_APPS_PATHS=$(occ_host config:system:get apps_paths --output json) + +if [ $REMOTE_ROOT_DIR ]; then + REAL_FEDERATED_SERVER_CONFIG_DIR="$REMOTE_ROOT_DIR/config" + REAL_FEDERATED_SERVER_DATA_DIR="$REMOTE_ROOT_DIR/data" + DESTROY_REAL_FEDERATED_SERVER=false +else echo '' - if [ $MAIN_SERVER_APPS_PATHS ]; then - echo -e "\033[0;33mCopying custom apps_paths\033[0m" - echo "{\"system\":{\"apps_paths\":$MAIN_SERVER_APPS_PATHS}}" > "$REAL_FEDERATED_SERVER_CONFIG_DIR/apps_paths.json" - NEXTCLOUD_CONFIG_DIR="$REAL_FEDERATED_SERVER_CONFIG_DIR" ${ROOT_DIR}/occ config:import < "$REAL_FEDERATED_SERVER_CONFIG_DIR/apps_paths.json" + echo -e "\033[0;36m#\033[0m" + echo -e "\033[0;36m# Setting up real federated server\033[0m" + echo -e "\033[0;36m#\033[0m" + REMOTE_ROOT_DIR=$ROOT_DIR + REAL_FEDERATED_SERVER_CONFIG_DIR="$MAIN_SERVER_DATA_DIR/tests-talk-real-federated-server/config" + REAL_FEDERATED_SERVER_DATA_DIR="$MAIN_SERVER_DATA_DIR/tests-talk-real-federated-server/data" + DESTROY_REAL_FEDERATED_SERVER=false + + if [ ! -d "$REAL_FEDERATED_SERVER_CONFIG_DIR" ] || occ_remote status | grep "installed: false"; then + DESTROY_REAL_FEDERATED_SERVER=true + if [ $CI ]; then + DESTROY_REAL_FEDERATED_SERVER=false + fi + echo '' + echo -e "\033[0;31mReal federated server not installed in $REAL_FEDERATED_SERVER_CONFIG_DIR\033[0m" + echo -e "\033[0;33mPerforming basic SQLite installation with data directory in $REAL_FEDERATED_SERVER_DATA_DIR\033[0m" + mkdir --parents "$REAL_FEDERATED_SERVER_CONFIG_DIR" "$REAL_FEDERATED_SERVER_DATA_DIR" + occ_remote maintenance:install --admin-pass=admin --data-dir="$REAL_FEDERATED_SERVER_DATA_DIR" echo '' + if [ $MAIN_SERVER_APPS_PATHS ]; then + echo -e "\033[0;33mCopying custom apps_paths\033[0m" + echo "{\"system\":{\"apps_paths\":$MAIN_SERVER_APPS_PATHS}}" > "$REAL_FEDERATED_SERVER_CONFIG_DIR/apps_paths.json" + occ_remote config:import < "$REAL_FEDERATED_SERVER_CONFIG_DIR/apps_paths.json" + echo '' + fi fi fi @@ -73,7 +93,7 @@ PORT_FED_REAL=8280 export PORT_FED_REAL echo "" > phpserver_fed_real.log -PHP_CLI_SERVER_WORKERS=3 NEXTCLOUD_CONFIG_DIR="$REAL_FEDERATED_SERVER_CONFIG_DIR" php -S localhost:${PORT_FED_REAL} -t ${ROOT_DIR} &> phpserver_fed_real.log & +PHP_CLI_SERVER_WORKERS=3 NEXTCLOUD_CONFIG_DIR="$REAL_FEDERATED_SERVER_CONFIG_DIR" php -S localhost:${PORT_FED_REAL} -t ${REMOTE_ROOT_DIR} &> phpserver_fed_real.log & PHPPID3=$! echo -e "Running on process ID: \033[1;35m$PHPPID3\033[0m" @@ -83,30 +103,29 @@ tail -f phpserver_fed_real.log | grep --line-buffered -v -E ":[0-9]+ Accepted$" # Kill all sub-processes in case of ctrl+c trap 'pkill -P $PHPPID1; pkill -P $PHPPID2; pkill -P $PHPPID3; pkill -P $PROCESS_ID; wait $PHPPID1; wait $PHPPID2; wait $PHPPID3;' INT TERM -NEXTCLOUD_ROOT_DIR=${ROOT_DIR} -export NEXTCLOUD_ROOT_DIR +export NEXTCLOUD_HOST_ROOT_DIR=${ROOT_DIR} +export NEXTCLOUD_HOST_CONFIG_DIR=${MAIN_SERVER_CONFIG_DIR} +export NEXTCLOUD_REMOTE_ROOT_DIR=${REMOTE_ROOT_DIR} +export NEXTCLOUD_REMOTE_CONFIG_DIR=${REAL_FEDERATED_SERVER_CONFIG_DIR} export TEST_SERVER_URL="http://localhost:8080/" export TEST_LOCAL_REMOTE_URL="http://localhost:8180/" export TEST_REMOTE_URL="http://localhost:8280/" -export MAIN_SERVER_CONFIG_DIR -export REAL_FEDERATED_SERVER_CONFIG_DIR export NEXTCLOUD_CONFIG_DIR="$MAIN_SERVER_CONFIG_DIR" -MAIN_OVERWRITE_CLI_URL=$(${ROOT_DIR}/occ config:system:get overwrite.cli.url) -MAIN_SKELETON_DIR=$(${ROOT_DIR}/occ config:system:get skeletondirectory) -${ROOT_DIR}/occ config:system:set overwrite.cli.url --value "http://localhost:8080/" +MAIN_OVERWRITE_CLI_URL=$(occ_host config:system:get overwrite.cli.url) +MAIN_SKELETON_DIR=$(occ_host config:system:get skeletondirectory) +occ_host config:system:set overwrite.cli.url --value "http://localhost:8080/" if [[ "$MAIN_SKELETON_DIR" != "" ]]; then echo "Resetting custom skeletondirectory so that tests pass" - ${ROOT_DIR}/occ config:system:delete skeletondirectory + ${NEXTCLOUD_HOST_OCC} config:system:delete skeletondirectory fi -export NEXTCLOUD_CONFIG_DIR="$REAL_FEDERATED_SERVER_CONFIG_DIR" -REAL_FEDERATED_OVERWRITE_CLI_URL=$(${ROOT_DIR}/occ config:system:get overwrite.cli.url) -REAL_FEDERATED_SKELETON_DIR=$(${ROOT_DIR}/occ config:system:get skeletondirectory) -${ROOT_DIR}/occ config:system:set overwrite.cli.url --value "$TEST_REMOTE_URL" +REAL_FEDERATED_OVERWRITE_CLI_URL=$(occ_remote config:system:get overwrite.cli.url) +REAL_FEDERATED_SKELETON_DIR=$(occ_remote config:system:get skeletondirectory) +occ_remote config:system:set overwrite.cli.url --value "$TEST_REMOTE_URL" if [[ "$REAL_FEDERATED_SKELETON_DIR" != "" ]]; then echo "Resetting custom skeletondirectory so that tests pass" - ${ROOT_DIR}/occ config:system:delete skeletondirectory + occ_remote config:system:delete skeletondirectory fi echo '' @@ -114,49 +133,50 @@ echo -e "\033[0;36m#\033[0m" echo -e "\033[0;36m# Setting up apps\033[0m" echo -e "\033[0;36m#\033[0m" cp -R ./spreedcheats ../../../spreedcheats -${ROOT_DIR}/occ app:getpath spreedcheats +occ_host app:getpath spreedcheats + +REMOTE_SPREED_DIR=$(occ_remote app:getpath spreed) +if [ ! -d ${REMOTE_ROOT_DIR}/apps/spreedcheats ]; then + cp -R ${REMOTE_SPREED_DIR}/tests/integration/spreedcheats ${REMOTE_ROOT_DIR}/apps/spreedcheats +fi # Add apps to the parent directory of "spreed" (unless they are # already there or in "apps"). -${ROOT_DIR}/occ app:getpath notifications || (cd ../../../ && git clone --depth 1 --branch ${NOTIFICATIONS_BRANCH} https://github.com/nextcloud/notifications) -${ROOT_DIR}/occ app:getpath guests || (cd ../../../ && git clone --depth 1 --branch ${GUESTS_BRANCH} https://github.com/nextcloud/guests) -${ROOT_DIR}/occ app:getpath circles || (cd ../../../ && git clone --depth 1 --branch ${CIRCLES_BRANCH} https://github.com/nextcloud/circles) -${ROOT_DIR}/occ app:getpath call_summary_bot || (cd ../../../ && git clone --depth 1 --branch ${CSB_BRANCH} https://github.com/nextcloud/call_summary_bot) - -for CONFIG_DIR in $MAIN_SERVER_CONFIG_DIR $REAL_FEDERATED_SERVER_CONFIG_DIR; do - export NEXTCLOUD_CONFIG_DIR="$CONFIG_DIR" - - ${ROOT_DIR}/occ app:enable spreed || exit 1 - ${ROOT_DIR}/occ app:enable --force spreedcheats || exit 1 - ${ROOT_DIR}/occ app:enable --force notifications || exit 1 - ${ROOT_DIR}/occ app:enable --force guests || exit 1 - ${ROOT_DIR}/occ app:enable --force circles || exit 1 - ${ROOT_DIR}/occ app:enable --force call_summary_bot || exit 1 - - ${ROOT_DIR}/occ app:list | grep spreed - ${ROOT_DIR}/occ app:list | grep notifications - ${ROOT_DIR}/occ app:list | grep guests - ${ROOT_DIR}/occ app:list | grep circles - ${ROOT_DIR}/occ app:list | grep call_summary_bot +occ_host app:getpath notifications || (cd ../../../ && git clone --depth 1 --branch ${NOTIFICATIONS_BRANCH} https://github.com/nextcloud/notifications) +occ_host app:getpath guests || (cd ../../../ && git clone --depth 1 --branch ${GUESTS_BRANCH} https://github.com/nextcloud/guests) +occ_host app:getpath circles || (cd ../../../ && git clone --depth 1 --branch ${CIRCLES_BRANCH} https://github.com/nextcloud/circles) +occ_host app:getpath call_summary_bot || (cd ../../../ && git clone --depth 1 --branch ${CSB_BRANCH} https://github.com/nextcloud/call_summary_bot) + +for OCC in occ_host occ_remote; do + ${OCC} app:enable spreed || exit 1 + ${OCC} app:enable --force spreedcheats || exit 1 + ${OCC} app:enable --force notifications || exit 1 + ${OCC} app:enable --force guests || exit 1 + ${OCC} app:enable --force circles || exit 1 + ${OCC} app:enable --force call_summary_bot || exit 1 + + ${OCC} app:list | grep spreed + ${OCC} app:list | grep notifications + ${OCC} app:list | grep guests + ${OCC} app:list | grep circles + ${OCC} app:list | grep call_summary_bot done echo '' echo -e "\033[0;36m#\033[0m" echo -e "\033[0;36m# Optimizing configuration\033[0m" echo -e "\033[0;36m#\033[0m" -for CONFIG_DIR in $MAIN_SERVER_CONFIG_DIR $REAL_FEDERATED_SERVER_CONFIG_DIR; do - export NEXTCLOUD_CONFIG_DIR="$CONFIG_DIR" - +for OCC in occ_host occ_remote; do # Disable bruteforce protection because the integration tests do trigger them - ${ROOT_DIR}/occ config:system:set auth.bruteforce.protection.enabled --value false --type bool + ${OCC} config:system:set auth.bruteforce.protection.enabled --value false --type bool # Disable rate limit protection because the integration tests do trigger them - ${ROOT_DIR}/occ config:system:set ratelimit.protection.enabled --value false --type bool + ${OCC} config:system:set ratelimit.protection.enabled --value false --type bool # Allow local remote urls otherwise we can not share - ${ROOT_DIR}/occ config:system:set allow_local_remote_servers --value true --type bool + ${OCC} config:system:set allow_local_remote_servers --value true --type bool # Enable debug mode as it is required to enable developer commands - ${ROOT_DIR}/occ config:system:set debug --value true --type bool + ${OCC} config:system:set debug --value true --type bool # Use faster password hashing - ${ROOT_DIR}/occ config:system:set hashing_default_password --value=true --type=bool + ${OCC} config:system:set hashing_default_password --value=true --type=bool done # Restore default config dir to local server in case it is used from the tests @@ -199,10 +219,10 @@ echo -e "\033[0;36m#\033[0m" # Main server export NEXTCLOUD_CONFIG_DIR="$MAIN_SERVER_CONFIG_DIR" -${ROOT_DIR}/occ app:disable spreedcheats -${ROOT_DIR}/occ config:system:set overwrite.cli.url --value "$MAIN_OVERWRITE_CLI_URL" +occ_host app:disable spreedcheats +occ_host config:system:set overwrite.cli.url --value "$MAIN_OVERWRITE_CLI_URL" if [[ "$MAIN_SKELETON_DIR" != "" ]]; then - ${ROOT_DIR}/occ config:system:set skeletondirectory --value "$MAIN_SKELETON_DIR" + occ_host config:system:set skeletondirectory --value "$MAIN_SKELETON_DIR" fi # Real federated server @@ -210,14 +230,15 @@ if $DESTROY_REAL_FEDERATED_SERVER; then rm -rf "$REAL_FEDERATED_SERVER_CONFIG_DIR" "$REAL_FEDERATED_SERVER_DATA_DIR" else export NEXTCLOUD_CONFIG_DIR="$REAL_FEDERATED_SERVER_CONFIG_DIR" - ${ROOT_DIR}/occ app:disable spreedcheats - ${ROOT_DIR}/occ config:system:set overwrite.cli.url --value "$REAL_FEDERATED_OVERWRITE_CLI_URL" + occ_remote app:disable spreedcheats + occ_remote config:system:set overwrite.cli.url --value "$REAL_FEDERATED_OVERWRITE_CLI_URL" if [[ "$REAL_FEDERATED_SKELETON_DIR" != "" ]]; then - ${ROOT_DIR}/occ config:system:set skeletondirectory --value "$REAL_FEDERATED_SKELETON_DIR" + occ_remote config:system:set skeletondirectory --value "$REAL_FEDERATED_SKELETON_DIR" fi fi rm -rf ../../../spreedcheats +rm -rf ${REMOTE_ROOT_DIR}/apps/spreedcheats wait $PHPPID1 wait $PHPPID2 From 7d619837cb06730d923e914cffce14101ef52254 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Dec 2024 09:00:41 +0100 Subject: [PATCH 3/5] test: Test both directions main <-> stable30 Signed-off-by: Joas Schilling --- .github/workflows/integration-federation.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/integration-federation.yml b/.github/workflows/integration-federation.yml index f36823c0ea3..b76fcab6407 100644 --- a/.github/workflows/integration-federation.yml +++ b/.github/workflows/integration-federation.yml @@ -45,6 +45,7 @@ jobs: needs: changes if: needs.changes.outputs.src != 'false' + name: Federation ${{ matrix.host-server-versions }} -> ${{ matrix.remote-server-versions }} strategy: fail-fast: false @@ -64,6 +65,20 @@ jobs: remote-circles-versions: 'stable30' remote-call-summary-bot-versions: 'main' remote-notifications-versions: 'stable30' + - test-suite: 'federation' + php-versions: '8.2' + host-server-versions: 'stable30' + host-spreed-versions: 'stable30' + host-guests-versions: 'stable30' + host-circles-versions: 'stable30' + host-call-summary-bot-versions: 'main' + host-notifications-versions: 'stable30' + remote-server-versions: 'master' + remote-spreed-versions: ${{ github.event.pull_request.head.sha }} + remote-guests-versions: 'master' + remote-circles-versions: 'master' + remote-call-summary-bot-versions: 'main' + remote-notifications-versions: 'master' services: oracle: From b919e01f7a50e846f1f49b15d609f5fd7d872b81 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Dec 2024 12:10:57 +0100 Subject: [PATCH 4/5] test: Test federation cross DBs Signed-off-by: Joas Schilling --- .github/workflows/integration-federation.yml | 90 ++++++++++++++++++-- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration-federation.yml b/.github/workflows/integration-federation.yml index b76fcab6407..2021628f651 100644 --- a/.github/workflows/integration-federation.yml +++ b/.github/workflows/integration-federation.yml @@ -45,7 +45,7 @@ jobs: needs: changes if: needs.changes.outputs.src != 'false' - name: Federation ${{ matrix.host-server-versions }} -> ${{ matrix.remote-server-versions }} + name: Federation ${{ matrix.host-server-versions }} (${{ matrix.host-db }}) -> ${{ matrix.remote-server-versions }} (${{ matrix.remote-db }}) strategy: fail-fast: false @@ -53,12 +53,14 @@ jobs: include: - test-suite: 'federation' php-versions: '8.2' + host-db: 'oci' host-server-versions: 'master' host-spreed-versions: ${{ github.event.pull_request.head.sha }} host-guests-versions: 'master' host-circles-versions: 'master' host-call-summary-bot-versions: 'main' host-notifications-versions: 'master' + remote-db: 'mysql' remote-server-versions: 'stable30' remote-spreed-versions: 'stable30' remote-guests-versions: 'stable30' @@ -67,12 +69,30 @@ jobs: remote-notifications-versions: 'stable30' - test-suite: 'federation' php-versions: '8.2' + host-db: 'pgsql' + host-server-versions: 'master' + host-spreed-versions: ${{ github.event.pull_request.head.sha }} + host-guests-versions: 'master' + host-circles-versions: 'master' + host-call-summary-bot-versions: 'main' + host-notifications-versions: 'master' + remote-db: 'mysql' + remote-server-versions: 'stable30' + remote-spreed-versions: 'stable30' + remote-guests-versions: 'stable30' + remote-circles-versions: 'stable30' + remote-call-summary-bot-versions: 'main' + remote-notifications-versions: 'stable30' + - test-suite: 'federation' + php-versions: '8.2' + host-db: 'oci' host-server-versions: 'stable30' host-spreed-versions: 'stable30' host-guests-versions: 'stable30' host-circles-versions: 'stable30' host-call-summary-bot-versions: 'main' host-notifications-versions: 'stable30' + remote-db: 'mysql' remote-server-versions: 'master' remote-spreed-versions: ${{ github.event.pull_request.head.sha }} remote-guests-versions: 'master' @@ -109,6 +129,16 @@ jobs: MYSQL_ROOT_PASSWORD: rootpassword options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + postgres: + image: ghcr.io/nextcloud/continuous-integration-postgres-14:latest + ports: + - 4445:5432/tcp + env: + POSTGRES_USER: root + POSTGRES_PASSWORD: rootpassword + POSTGRES_DB: nextcloud + options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 + steps: - name: Set app env run: | @@ -224,13 +254,63 @@ jobs: working-directory: host/apps/${{ env.APP_NAME }}/tests/integration run: composer i - - name: Set up Nextcloud - Host + - name: Set up Nextcloud - Host (Postgres) + if: ${{ matrix.host-db == 'pgsql' }} + env: + DB_PORT: 4445 + working-directory: host/ + run: | + mkdir data + ./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + + - name: Set up Nextcloud - Host (Oracle) + if: ${{ matrix.host-db == 'oci' }} env: DB_PORT: 1521 working-directory: host/ run: | mkdir data ./occ maintenance:install --verbose --database=oci --database-name=XE --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=autotest --database-pass=owncloud --admin-user admin --admin-pass admin + + - name: Set up Nextcloud - Host (MySQL) + if: ${{ matrix.host-db == 'mysql' }} + env: + DB_PORT: 4444 + working-directory: host/ + run: | + mkdir data + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + + - name: Set up Nextcloud - Remote (Postgres) + if: ${{ matrix.remote-db == 'pgsql' }} + env: + DB_PORT: 4445 + working-directory: remote/ + run: | + mkdir data + ./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + + - name: Set up Nextcloud - Remote (Oracle) + if: ${{ matrix.remote-db == 'oracle' }} + env: + DB_PORT: 1521 + working-directory: remote/ + run: | + mkdir data + ./occ maintenance:install --verbose --database=oci --database-name=XE --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=autotest --database-pass=owncloud --admin-user admin --admin-pass admin + + - name: Set up Nextcloud - Remote (MySQL) + if: ${{ matrix.remote-db == 'mysql' }} + env: + DB_PORT: 4444 + working-directory: remote/ + run: | + mkdir data + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + + - name: Set up Nextcloud - Host (Shared) + working-directory: host/ + run: | ./occ config:system:set debug --value=true --type=boolean ./occ config:system:set hashing_default_password --value=true --type=boolean ./occ config:system:set memcache.local --value="\\OC\\Memcache\\APCu" @@ -241,13 +321,9 @@ jobs: ./occ app:enable --force guests ./occ app:enable --force notifications - - name: Set up Nextcloud - Remote - env: - DB_PORT: 4444 + - name: Set up Nextcloud - Remote (Shared) working-directory: remote/ run: | - mkdir data - ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin ./occ config:system:set debug --value=true --type=boolean ./occ config:system:set hashing_default_password --value=true --type=boolean ./occ config:system:set memcache.local --value="\\OC\\Memcache\\APCu" From df0be97be4fcbb6fd8aac107963adf57361a63dd Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 10 Dec 2024 14:47:03 +0100 Subject: [PATCH 5/5] test: Print output when job fails Signed-off-by: Joas Schilling --- .../integration/features/bootstrap/FeatureContext.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/features/bootstrap/FeatureContext.php b/tests/integration/features/bootstrap/FeatureContext.php index 3167b495f6d..c12849ba46b 100644 --- a/tests/integration/features/bootstrap/FeatureContext.php +++ b/tests/integration/features/bootstrap/FeatureContext.php @@ -4877,7 +4877,15 @@ public function userShareLastNotificationFile(string $user, string $firstLast, s */ public function runReminderBackgroundJobs(string $useForce, string $class, bool $repeated = false): void { $this->runOcc(['background-job:list', '--output=json_pretty', '--class=' . $class]); - $list = json_decode($this->lastStdOut, true, 512, JSON_THROW_ON_ERROR); + try { + $list = json_decode($this->lastStdOut, true, 512, JSON_THROW_ON_ERROR); + } catch (JsonException $e) { + var_dump('Output'); + var_dump($this->lastStdOut); + var_dump('Error'); + var_dump($this->lastStdErr); + throw $e; + } if ($repeated && empty($list)) { return;