internal: add CI checks for major Linux distros and fix build errors (#…

…2276)

.github/workflows/pr-ubuntu-docker.yml → .github/workflows/pr-linux-variants.yml

@@ -1,4 +1,6 @@
-name: PR Ubuntu Docker
+# TODO(#2280): rename from PR to something else (also applies to Windows etc)
+# TODO(#2280): keep in line with deploy.yml
+name: PR Linux
 on:
     pull_request:
         # any
@@ -10,20 +12,115 @@ env:
     BUILD_TYPE: RelWithDebInfo
 
 jobs:
-    pr-ubuntu-docker:
+    pr-docker:
+        strategy:
+          matrix:
+            base_system:
+            # The usual container used for all builds within the CI and the one
+            # we use as the base for server builds with preinstalled/cached deps
+            - ubuntu-vcpkg-deps
+
+            # Distros that our contributors are likely to run
+            - ubuntu-2204
+            - ubuntu-2404
+            - ubuntu-2410  # latest non-LTS
+            - debian-12
+            - archlinux
+
+            # TODO(#2280): add Fedora
+            # TODO(#2280): test gcc build, test build w/o ninja
+
         runs-on: ubuntu-latest
         steps:
           - name: Checkout
             uses: actions/checkout@v4
             with:
                 submodules: recursive
 
-          - name: Get image to build upon
+          - name: Debug - print env
+            run: env
+
+          - name: Get/build image to use for building & testing
             run: |
-                cat ${{github.workspace}}/misc/github_env_linux >> "$GITHUB_ENV"
+                set -x  # print executed commands
+
+                echo Some debug data
+                id
+                uname -a
+                lsb_release -a
+
+                # cat ${{github.workspace}}/misc/github_env_linux >> "$GITHUB_ENV"
+
+                # get base image for this distro (or build configuration, in case with skymp-deps)
+                base_image="`misc/deps_linux/${{ matrix.base_system }}.sh --get-image-uri`"
+
+                docker run \
+                  --name prepare_deps_container \
+                  -v "${{github.workspace}}:/src" \
+                  -w "/src" \
+                  -e "CI=$CI" \
+                  -e "CREATE_UID=`id -u`" \
+                  "$base_image" \
+                  'misc/deps_linux/${{ matrix.base_system }}.sh' '--ensure-deps-noninteractive'
+
+                docker commit prepare_deps_container localhost/buildimage
+
+          - name: Debug - check build tools
+            uses: addnab/docker-run-action@v3
+            with:
+              image: localhost/buildimage
+              options: |
+                -v ${{github.workspace}}:/src
+                -w /src
+                -u skymp
+              run: |
+                print_info_unsafe() {
+                  echo .
+                  echo .
+                  echo "=== Checking $1 ==="
+                  out="`which $1`"
+                  ret="$?"
+                  if [ "$ret" != 0 -o -z "$out" ]; then
+                    echo "Which exited with $ret $out"
+                    return
+                  fi
+                  ls -lh "$out"
+                  echo "version call:"
+                  "$out" --version
+                }
+
+                print_info() {
+                  print_info_unsafe "$@" || true
+                }
+
+                print_info clang++
+                print_info clang++-15
+                print_info cmake
+                print_info ninja
+                print_info node
+                print_info yarn
+                print_info git
+                print_info zip
+                print_info unzip
+                print_info curl
+
+                echo .
+                echo .
+                echo === misc ===
+                echo "current dir is $PWD"
+                echo /src:
+                ls -lh /src
+
+                echo .
+                echo .
+                echo === env ===
+                ./build.sh --print-env
 
           - name: Store SP types from commit
             run: |
+              id
+              ls -lh ${{github.workspace}}/skyrim-platform/src/platform_se/codegen/convert-files
+
               commit=$(< ${{github.workspace}}/skyrim-platform/src/platform_se/codegen/convert-files/skyrimPlatform.ts)
               chmod 777 ${{github.workspace}}/skyrim-platform/src/platform_se/codegen/convert-files
               echo "$commit" > ${{github.workspace}}/skyrim-platform/src/platform_se/codegen/convert-files/temp.txt
@@ -58,7 +155,7 @@ jobs:
           - name: Prepare
             uses: addnab/docker-run-action@v3
             with:
-                image: ${{ env.SKYMP_VCPKG_DEPS_IMAGE }}
+                image: localhost/buildimage
                 options: |
                     -v ${{github.workspace}}:/src
                 run: |
@@ -68,7 +165,7 @@ jobs:
             id: cmake_configure
             uses: addnab/docker-run-action@v3
             with:
-                image: ${{ env.SKYMP_VCPKG_DEPS_IMAGE }}
+                image: localhost/buildimage
                 options: |
                     -v ${{github.workspace}}:/src
                     -u skymp
@@ -78,23 +175,25 @@ jobs:
                        -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \
                        -DUNIT_DATA_DIR="/src/skyrim_data_files"
 
-          - name: Upload vcpkg failure logs
-            if: failure()
+          - name: Upload vcpkg logs
+            if: always()
             uses: actions/upload-artifact@v4
             with:
-              name: install-x64-linux-dbg-out.log
-              path: ${{github.workspace}}/vcpkg/buildtrees/rsm-bsa/install-x64-linux-dbg-out.log
+              name: ${{matrix.base_system}}_vcpkg_logs
+              path: |
+                ${{github.workspace}}/build/vcpkg-bootstrap.log
+                ${{github.workspace}}/vcpkg/buildtrees/**/*.log
 
           - name: Upload compile_commands.json
             uses: actions/upload-artifact@v4
             with:
-                name: compile_commands.json
+                name: ${{matrix.base_system}}_compile_commands.json
                 path: ${{github.workspace}}/build/compile_commands.json
 
           - name: Build
             uses: addnab/docker-run-action@v3
             with:
-                image: ${{ env.SKYMP_VCPKG_DEPS_IMAGE }}
+                image: localhost/buildimage
                 options: |
                     -v ${{github.workspace}}:/src
                     -u skymp
@@ -105,7 +204,7 @@ jobs:
           - name: Prepare dist.tar.gz
             uses: addnab/docker-run-action@v3
             with:
-                image: ${{ env.SKYMP_VCPKG_DEPS_IMAGE }}
+                image: localhost/buildimage
                 options: |
                     -v ${{github.workspace}}:/src
                     -u skymp
@@ -116,7 +215,7 @@ jobs:
           - name: Upload dist.tar.gz
             uses: actions/upload-artifact@v4
             with:
-              name: dist.tar.gz
+              name: ${{matrix.base_system}}_dist.tar.gz
               path: ${{github.workspace}}/build/dist.tar.gz
 
           - name: SP Types Check
@@ -148,7 +247,7 @@ jobs:
                 echo "ConstEnumApi.cpp has changed after running const_enum_extractor. Please re-run it locally and commit changes"
                 exit 1
               fi
-          
+
           - name: SP Types Check failed - Here is what you can do
             if: ${{ failure() && steps.sp_types_check.outcome == 'failure' }}
             run: |
@@ -164,7 +263,7 @@ jobs:
           - name: Test
             uses: addnab/docker-run-action@v3
             with:
-                image: ${{ env.SKYMP_VCPKG_DEPS_IMAGE }}
+                image: localhost/buildimage
                 options: |
                     -v ${{github.workspace}}:/src
                     -u skymp

build.sh

@@ -15,34 +15,25 @@ fi
 
 export VCPKG_DISABLE_METRICS=1
 
-export CC=/usr/bin/clang-15
-export CPP=/usr/bin/clang-cpp-15
-export CXX=/usr/bin/clang++-15
-export LD=/usr/bin/ld.lld-15
+if which clang++ > /dev/null 2>&1; then
+  export CC=/usr/bin/clang
+  export CPP=/usr/bin/clang-cpp
+  export CXX=/usr/bin/clang++
+else
+  export CC=/usr/bin/clang-15
+  export CPP=/usr/bin/clang-cpp-15
+  export CXX=/usr/bin/clang++-15
+fi
 
 export CMAKE_C_COMPILER="$CC"
 export CMAKE_CXX_COMPILER="$CXX"
-export CMAKE_MAKE_PROGRAM="ninja"
-
-# Some build dependencies require some strange stuff.
-# Some other deps won't work if we can't call clang without exact version.
-# It's better to make compatibility aliases than breaking main system's root.
-export SKYMP_COMPAT_BIN="$PWD/build/skymp_compat/bin"
-export PATH="$SKYMP_COMPAT_BIN:$PATH"
+export CMAKE_MAKE_PROGRAM="/usr/bin/ninja"
 
 if [ ! -d build ]; then
   mkdir -v build
 fi
 
-if [ ! -d "$SKYMP_COMPAT_BIN" ]; then
-  mkdir -pv "$SKYMP_COMPAT_BIN"
-  ln -s "$CC" "$SKYMP_COMPAT_BIN/clang"
-  ln -s "$CXX" "$SKYMP_COMPAT_BIN/clang++"
-
-  echo "Set up compatibility path for build."
-fi
-
-# TODO reverse the order or use [[ ]] ?
+# TODO(#2280): reverse the order or use [[ ]] ?
 if [ "$1" = "--configure" ]; then
   shift && \
     cd build && \
@@ -52,18 +43,36 @@ elif [ "$1" = "--build" ]; then
     cd build && \
     exec cmake --build . "$@"
 elif [ "$1" = "--clean" ]; then
-  eecho "NOTE: --clean was deprecated, please use --clean-cpp or --clean-node"
+  eecho "NOTE: --clean was deprecated, please use one of the options listed in --help"
   eecho "Proceeding with cleaning cpp build data"
   exec rm -rf build/
 elif [ "$1" = "--clean-cpp" ]; then
   exec rm -rf build/
 elif [ "$1" = "--clean-node" ]; then
-  echo not implemented yet, TODO delete node_modules dirs
+  find -name node_modules -type d -prune -print0 | xargs -0 echo rm -rf
+  echo -n Ctrl-C to abort, Return/Enter to proceed
+  read
+  find -name node_modules -type d -prune -print0 | xargs --verbose -0 rm -rf
+elif [ "$1" = "--clean-vcpkg" ]; then
+  (cd vcpkg && git clean -xfd --dry-run)
+  echo -n Ctrl-C to abort, Return/Enter to proceed
+  read
+  (cd vcpkg && git clean -xfd)
+elif [ "$1" = "--print-env" ]; then
+  env
 else
   eecho "Usage:"
   eecho "  ./build.sh --configure <cmake args...>"
+  eecho "  runs in build: cmake .. <some extra args> <your args>"
   eecho "OR"
   eecho "  ./build.sh --build <cmake args...>"
+  eecho "  runs in build: cmake build . <your args>"
+  eecho "  you can add args like --target=unit"
+  eecho "OR"
+  eecho "  ./build.sh --clean-<cpp|node|vcpkg>"
   eecho "OR"
-  eecho "  ./build.sh --clean-<cpp|node>"
+  eecho "  ./build.sh --print-env"
+  if [ "$1" != "--help" ]; then
+    exit 1
+  fi
 fi

cmake/yarn.cmake

@@ -34,7 +34,7 @@ function(yarn_execute_command)
 
   if("${A_RESULT_VARIABLE}" STREQUAL "")
     if(NOT "${yarn_result}" STREQUAL "0")
-      message(FATAL_ERROR "yarn ${A_COMMAND} exited with ${yarn_result}:\n${yarn_output}\n${yarn_error}")
+      message(FATAL_ERROR "yarn ${A_COMMAND} (working dir ${A_WORKING_DIRECTORY}) exited with ${yarn_result}:\n${yarn_output}\n${yarn_error}")
     endif()
   else()
     set("${A_RESULT_VARIABLE}" "${yarn_result}" PARENT_SCOPE)

libespm/src/WRLD.cpp

@@ -1,4 +1,7 @@
 #include "libespm/WRLD.h"
+
+#include <algorithm>
+
 #include "libespm/RecordHeaderAccess.h"
 
 namespace espm {

misc/deps_linux/archlinux.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+# Scroll down for the list of packages
+
+set -e
+
+if [[ "$1" == "--get-image-uri" ]]; then
+  # This is the image that we'll use in our CI for testing build for this distro
+  echo 'archlinux:base-devel'
+  exit
+fi
+
+if [[ "$1" == "--ensure-deps-noninteractive" ]]; then
+  if [[ -z "$CI" ]]; then
+    echo "This isn't CI environment! Are you sure you didn't want to use just --ensure-deps?"
+    exit 1
+  fi
+  set -x
+  PACMAN_ARGS="--noconfirm"
+  pacman -Syu $PACMAN_ARGS sudo
+  useradd -m skymp -u $CREATE_UID
+  chown -R skymp:skymp /src
+
+  cat /etc/passwd
+
+  DO_ENSURE_DEPS=1
+fi
+
+if [[ "$1" == "--ensure-deps" ]]; then
+  DO_ENSURE_DEPS=1
+  PACMAN_ARGS="--confirm"
+fi
+
+if [[ -z "$DO_ENSURE_DEPS" ]]; then
+  echo "idk what to do"
+  exit 1
+fi
+
+packages=''
+
+addpackage() {
+  packages+="$1 "
+}
+
+# These are some basic packages that are essential for C++ build config that we're using
+addpackage base-devel
+addpackage cmake
+addpackage clang
+addpackage ninja  # would likely build with the regular make, but I haven't tried
+# gdb is also recommended for debugging but isn't required
+
+# These are needed for some parts of the client and server, as well as some build scripts
+addpackage nodejs  # 23 as of 20241222
+# alternatively, nodejs-lts-iron for 20 or nodejs-lts-hydrogen for 18
+# I only found 20, not 22 - https://archlinux.org/packages/?sort=&q=nodejs&maintainer=&flagged=
+addpackage yarn
+
+# Some packages that you likely already have, but we'll just make sure
+addpackage git
+addpackage zip
+addpackage unzip
+addpackage tar
+addpackage curl  # required by vcpkg
+addpackage pkgconf  # required by Catch2 (C++ testing framework)
+addpackage linux-headers  # required by OpenSSL(?)
+
+echo Will now run the installation command, please check it and confirm
+set -x  # this will print the list that we're going to install
+sudo pacman -S --needed $PACMAN_ARGS $packages