Explicit Update with Indices implementation

.ci/generate-buildkite-pipeline-premerge

@@ -233,10 +233,7 @@ linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
 
 windows_projects_to_test=$(exclude-windows $(compute-projects-to-test ${modified_projects}))
 windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
-# Temporary disable the windows job.
-# See https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840
-#windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
-windows_projects=""
+windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
 
 # Generate the appropriate pipeline
 if [[ "${linux_projects}" != "" ]]; then

.ci/monolithic-windows.sh

@@ -38,6 +38,12 @@ targets="${2}"
 
 echo "--- cmake"
 pip install -q -r ${MONOREPO_ROOT}/mlir/python/requirements.txt
+
+# The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
+# on fixing a build reliability issue on the build server, please
+# see https://github.com/llvm/llvm-project/pull/82393 and
+# https://discourse.llvm.org/t/rfc-future-of-windows-pre-commit-ci/76840/40
+# for further information.
 cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D LLVM_ENABLE_PROJECTS="${projects}" \
       -G Ninja \
@@ -49,7 +55,10 @@ cmake -S ${MONOREPO_ROOT}/llvm -B ${BUILD_DIR} \
       -D COMPILER_RT_BUILD_ORC=OFF \
       -D CMAKE_C_COMPILER_LAUNCHER=sccache \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
-      -D MLIR_ENABLE_BINDINGS_PYTHON=ON
+      -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
+      -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
+      -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
+      -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO"
 
 echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.

.github/new-prs-labeler.yml

@@ -822,6 +822,30 @@ backend:X86:
   - llvm/lib/TargetParser/X86*
   - llvm/utils/TableGen/X86*
 
+backend:PowerPC:
+  - llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC*
+  - llvm/include/llvm/BinaryFormat/XCOFF.h
+  - llvm/include/llvm/IR/IntrinsicsPowerPC.td
+  - llvm/lib/CodeGen/AsmPrinter/AIXException.cpp
+  - llvm/lib/Target/PowerPC/**
+  - llvm/test/Analysis/**/PowerPC/**
+  - llvm/test/CodeGen/PowerPC/**
+  - llvm/test/CodeGen/MIR/PowerPC/**
+  - llvm/test/DebugInfo/XCOFF/**
+  - llvm/test/DebugInfo/PowerPC/**
+  - llvm/test/LTO/PowerPC/**
+  - llvm/test/MC/Disassembler/PowerPC/**
+  - llvm/test/MC/PowerPC/**
+  - llvm/test/MC/XCOFF/**
+  - llvm/test/Transforms/**/PowerPC/**
+  - clang/include/clang/Basic/BuiltinsPPC.*
+  - clang/lib/Basic/Targets/PPC.*
+  - clang/lib/CodeGen/Targets/PPC.cpp
+  - clang/lib/Driver/ToolChains/PPC*
+  - clang/lib/Driver/ToolChains/AIX*
+  - clang/lib/Driver/ToolChains/Arch/PPC.*
+  - clang/test/CodeGen/PowerPC/**
+
 third-party:unittests:
   - third-party/unittests/**
 

.github/workflows/docs.yml

@@ -28,6 +28,7 @@ on:
       - 'polly/docs/**'
       - 'flang/docs/**'
       - 'flang/include/flang/Optimizer/Dialect/FIROps.td'
+      - '.github/workflows/docs.yml'
   pull_request:
     paths:
       - 'llvm/docs/**'
@@ -45,6 +46,7 @@ on:
       - 'polly/docs/**'
       - 'flang/docs/**'
       - 'flang/include/flang/Optimizer/Dialect/FIROps.td'
+      - '.github/workflows/docs.yml'
 
 jobs:
   check-docs-build:
@@ -162,8 +164,6 @@ jobs:
           TZ=UTC ninja -C polly-build docs-polly-html docs-polly-man
       - name: Build Flang docs
         if: steps.docs-changed-subprojects.outputs.flang_any_changed == 'true'
-        # TODO(boomanaiden154): Remove the SPHINX_WARNINGS_AS_ERRORS from the
-        # CMake invocation once the warnings in the flang docs build are fixed.
         run: |
-          cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON -DSPHINX_WARNINGS_AS_ERRORS=OFF ./llvm
-          TZ=UTC ninja -C flang-build docs-flang-html docs-flang-man
+          cmake -B flang-build -GNinja -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_PROJECTS="clang;mlir;flang" -DLLVM_ENABLE_SPHINX=ON ./llvm
+          TZ=UTC ninja -C flang-build docs-flang-html

.github/workflows/llvm-project-tests.yml

@@ -58,6 +58,10 @@ jobs:
   lit-tests:
     name: Lit Tests
     runs-on: ${{ matrix.os }}
+    container:
+      image: ${{(startsWith(matrix.os, 'ubuntu') && 'ghcr.io/llvm/ci-ubuntu-22.04:latest') || null}}
+      volumes:
+        - /mnt/:/mnt/
     strategy:
       fail-fast: false
       matrix:
@@ -77,6 +81,7 @@ jobs:
         with:
           python-version: ${{ inputs.python_version }}
       - name: Install Ninja
+        if: runner.os != 'Linux'
         uses: llvm/actions/install-ninja@main
       # actions/checkout deletes any existing files in the new git directory,
       # so this needs to either run before ccache-action or it has to use
@@ -108,8 +113,8 @@ jobs:
         run: |
           if [ "${{ runner.os }}" == "Linux" ]; then
             builddir="/mnt/build/"
-            sudo mkdir -p $builddir
-            sudo chown `whoami`:`whoami` $builddir
+            mkdir -p $builddir
+            extra_cmake_args="-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang"
           else
             builddir="$(pwd)"/build
           fi
@@ -123,6 +128,7 @@ jobs:
                 -DLLDB_INCLUDE_TESTS=OFF \
                 -DCMAKE_C_COMPILER_LAUNCHER=sccache \
                 -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+                $extra_cmake_args \
                 ${{ inputs.extra_cmake_args }}
           ninja -C "$builddir" '${{ inputs.build_target }}'
 

.github/workflows/release-tasks.yml

@@ -28,6 +28,7 @@ jobs:
     name: Create a New Release
     runs-on: ubuntu-latest
     needs: validate-tag
+
     steps:
       - name: Install Dependencies
         run: |
@@ -40,8 +41,9 @@ jobs:
       - name: Create Release
         env:
           GITHUB_TOKEN: ${{ github.token }}
+          USER_TOKEN: ${{ secrets.RELEASE_TASKS_USER_TOKEN }}
         run: |
-          ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --release ${{ needs.validate-tag.outputs.release-version }} --user ${{ github.actor }} create
+          ./llvm/utils/release/./github-upload-release.py --token "$GITHUB_TOKEN" --release ${{ needs.validate-tag.outputs.release-version }} --user ${{ github.actor }} --user-token "$USER_TOKEN" create
   release-documentation:
     name: Build and Upload Release Documentation
     needs:

.github/workflows/sycl-detect-changes.yml

@@ -6,6 +6,9 @@ on:
       filters:
         description: Matched filters
         value: ${{ jobs.need_check.outputs.filters }}
+      arc_tests:
+        description: Tests to run on Arc
+        value: ${{ jobs.need_check.outputs.arc_tests }}
 
 jobs:
   need_check:
@@ -16,6 +19,7 @@ jobs:
     timeout-minutes: 3
     outputs:
       filters: ${{ steps.result.outputs.result }}
+      arc_tests: ${{ steps.arc_tests.outputs.arc_tests }}
     steps:
       - name: Check file changes
         uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd
@@ -57,6 +61,19 @@ jobs:
               - devops/scripts/install_drivers.sh
             perf-tests:
               - sycl/test-e2e/PerformanceTests/**
+            esimd:
+              - 'llvm/lib/SYCLLowerIR/ESIMD/**'
+              - 'llvm/lib/SYCLLowerIR/LowerInvokeSimd.cpp'
+              - 'llvm/include/llvm/SYCLLowerIR/LowerInvokeSimd.h'
+              - 'sycl/include/std/experimental/simd.hpp'
+              - 'sycl/include/std/experimental/simd.hpp'
+              - 'sycl/include/sycl/ext/intel/esimd.hpp'
+              - 'sycl/include/sycl/ext/intel/esimd/**'
+              - 'sycl/include/sycl/ext/intel/esimd.hpp'
+              - 'sycl/include/sycl/ext/intel/experimental/esimd/**'
+              - 'sycl/include/sycl/ext/oneapi/experimental/invoke_simd.hpp'
+              - 'sycl/include/sycl/ext/oneapi/experimental/detail/invoke_simd_types.hpp'
+              - 'sycl/test-e2e/(ESIMD|InvokeSimd)/**'
 
       - name: Set output
         id: result
@@ -69,6 +86,15 @@ jobs:
               return '${{ steps.changes.outputs.changes }}';
             }
             // Treat everything as changed for huge PRs.
-            return ["llvm", "llvm_spirv", "clang", "sycl_fusion", "xptifw", "libclc", "sycl", "ci"];
+            return ["llvm", "llvm_spirv", "clang", "sycl_fusion", "xptifw", "libclc", "sycl", "ci", "esimd"];
 
       - run: echo '${{ steps.result.outputs.result }}'
+
+      - name: Set Arc tests
+        id: arc_tests
+        run: |
+          if [ "${{ contains(steps.result.outputs.result, 'esimd') }}" == "true" ]; then
+            echo 'arc_tests="(ESIMD|InvokeSimd|Matrix)/"' >> "$GITHUB_OUTPUT"
+          else
+            echo 'arc_tests="Matrix/"' >> "$GITHUB_OUTPUT"
+          fi

.github/workflows/sycl-linux-precommit.yml

@@ -68,15 +68,15 @@ jobs:
             reset_gpu: true
             install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
             extra_lit_opts: --param gpu-intel-gen12=True
-          - name: Matrix E2E tests on Intel Arc A-Series Graphics
+          - name: E2E tests on Intel Arc A-Series Graphics
             runner: '["Linux", "arc"]'
             image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
             image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
             target_devices: ext_oneapi_level_zero:gpu;opencl:gpu
             reset_gpu: true
             install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }}
             extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True
-            env: '{"LIT_FILTER":"Matrix/"}'
+            env: '{"LIT_FILTER":${{ needs.detect_changes.outputs.arc_tests }} }'
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     with:
       name: ${{ matrix.name }}
@@ -109,21 +109,37 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: Perf tests on Intel GEN12 Graphics system
+          - name: Intel GEN12 Graphics system
             runner: '["Linux", "gen12"]'
-          - name: Perf tests on Intel Arc A-Series Graphics system
+            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image_extra_opts: --device=/dev/dri
+            reset_gpu: true
+          - name: Intel Arc A-Series Graphics system
             runner: '["Linux", "arc"]'
+            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image_extra_opts: --device=/dev/dri
+            reset_gpu: true
+          - name: AMD system
+            runner: '["Linux", "amdgpu"]'
+            image: ghcr.io/intel/llvm/ubuntu2204_build:latest
+            image_extra_opts: --device=/dev/dri --device=/dev/kfd
+            extra_cmake_args: -DHIP_PLATFORM="AMD" -DAMD_ARCH="gfx1031"
+          - name: CUDA system
+            runner: '["Linux", "cuda"]'
+            image: ghcr.io/intel/llvm/ubuntu2204_build:latest
+            image_extra_opts: --gpus all
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     with:
-      name: ${{ matrix.name }}
+      name: Perf tests on ${{ matrix.name }}
       runner: ${{ matrix. runner }}
-      image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
-      image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+      image: ${{ matrix.image }}
+      image_options: -u 1001 --privileged --cap-add SYS_ADMIN ${{ matrix.image_extra_opts }}
       target_devices: all
-      reset_gpu: true
+      reset_gpu: ${{ matrix.reset_gpu }}
 
       env: '{"LIT_FILTER":"PerformanceTests/"}'
       extra_lit_opts: -a -j 1 --param enable-perf-tests=True
+      extra_cmake_args: ${{ matrix.extra_cmake_args }}
 
       ref: ${{ github.sha }}
       merge_ref: ''

.github/workflows/sycl-linux-run-tests.yml

@@ -81,6 +81,7 @@ on:
           - '["Linux", "gen12"]'
           - '["amdgpu"]'
           - '["Linux", "arc"]'
+          - '["cts-cpu"]'
       image:
         description: |
           Use option ending with ":build" for AMDGPU, ":latest" for the rest.
@@ -286,10 +287,27 @@ jobs:
         -DDPCPP_INSTALL_DIR="$$(dirname (which clang++))/.." \
         $CMAKE_EXTRA_ARGS
         ninja -C build-cts
+
+    - name: SYCL CTS List devices
+      if: inputs.tests_selector == 'cts'
+      run: |
+        ./build-cts/bin/test_all --list-devices
+
     - name: Run SYCL CTS tests
       if: inputs.tests_selector == 'cts'
       env:
         ONEAPI_DEVICE_SELECTOR: ${{ inputs.target_devices }}
+      # FIXME: For some reason the "sub_group api" test-case is failing with
+      # SIGSEGV while running test_all, so running each binary separately.
+      # BTW test_all requires a lot of resources to build it, so probably it'll
+      # be better to build each binary also separately.
+      # run: |
+      #   ./build-cts/bin/test_all $CTS_TEST_ARGS
       run: |
-        ./build-cts/bin/test_all --list-devices
-        ./build-cts/bin/test_all $CTS_TEST_ARGS
+        for i in `ls -1 ./build-cts/bin`; do
+          if [ "$i" != "test_all" ]; then
+            echo "::group::Running $i"
+            build-cts/bin/$i
+            echo "::endgroup::"
+          fi
+        done

.github/workflows/sycl-nightly.yml

@@ -71,13 +71,21 @@ jobs:
             image: ghcr.io/intel/llvm/ubuntu2204_build:latest
             image_options: -u 1001 --gpus all --cap-add SYS_ADMIN
             target_devices: ext_oneapi_cuda:gpu
+
+          - name: SYCL-CTS
+            runner: '["cts-cpu"]'
+            image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest
+            image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN
+            target_devices: opencl:cpu
+            tests_selector: cts
     uses: ./.github/workflows/sycl-linux-run-tests.yml
     with:
       name: ${{ matrix.name }}
       runner: ${{ matrix.runner }}
       image: ${{ matrix.image }}
       image_options: ${{ matrix.image_options }}
       target_devices: ${{ matrix.target_devices }}
+      tests_selector: ${{ matrix.tests_selector }}
       reset_gpu: ${{ matrix.reset_gpu }}
       ref: ${{ github.sha }}
       merge_ref: ''

bolt/include/bolt/Core/BinarySection.h

@@ -139,10 +139,7 @@ class BinarySection {
     Alignment = NewAlignment;
     ELFType = NewELFType;
     ELFFlags = NewELFFlags;
-    OutputSize = NewSize;
-    OutputContents = StringRef(reinterpret_cast<const char *>(NewData),
-                               NewData ? NewSize : 0);
-    IsFinalized = true;
+    updateContents(NewData, NewSize);
   }
 
 public:
@@ -484,9 +481,18 @@ class BinarySection {
   void flushPendingRelocations(raw_pwrite_stream &OS,
                                SymbolResolverFuncTy Resolver);
 
-  /// Change contents of the section.
-  void updateContents(const uint8_t *Data, size_t NewSize) {
-    OutputContents = StringRef(reinterpret_cast<const char *>(Data), NewSize);
+  /// Change contents of the section. Unless the section has a valid SectionID,
+  /// the memory passed in \p NewData will be managed by the instance of
+  /// BinarySection.
+  void updateContents(const uint8_t *NewData, size_t NewSize) {
+    if (getOutputData() && !hasValidSectionID() &&
+        (!hasSectionRef() ||
+         OutputContents.data() != getContentsOrQuit(Section).data())) {
+      delete[] getOutputData();
+    }
+
+    OutputContents = StringRef(reinterpret_cast<const char *>(NewData),
+                               NewData ? NewSize : 0);
     OutputSize = NewSize;
     IsFinalized = true;
   }

bolt/include/bolt/Core/DIEBuilder.h

@@ -124,7 +124,7 @@ class DIEBuilder {
   std::vector<std::unique_ptr<DIEAbbrev>> Abbreviations;
   BinaryContext &BC;
   DWARFContext *DwarfContext{nullptr};
-  bool IsDWO{false};
+  DWARFUnit *SkeletonCU{nullptr};
   uint64_t UnitSize{0};
   llvm::DenseSet<uint64_t> AllProcessed;
 
@@ -264,8 +264,12 @@ class DIEBuilder {
   /// current Section.
   DIE *constructDIEFast(DWARFDie &DDie, DWARFUnit &U, uint32_t UnitId);
 
+  /// Returns true if this DIEBUilder is for DWO Unit.
+  bool isDWO() const { return SkeletonCU != nullptr; }
+
 public:
-  DIEBuilder(BinaryContext &BC, DWARFContext *DwarfContext, bool IsDWO = false);
+  DIEBuilder(BinaryContext &BC, DWARFContext *DwarfContext,
+             DWARFUnit *SkeletonCU = nullptr);
 
   /// Returns enum to what we are currently processing.
   ProcessingType getCurrentProcessingState() { return getState().Type; }