LLVM and SPIRV-LLVM-Translator pulldown (WW06)

.github/workflows/build-ci-container.yml

@@ -1,4 +1,3 @@
-
 name: Build CI Container
 
 permissions:
@@ -19,9 +18,41 @@ on:
       - '.github/workflows/containers/github-action-ci/**'
 
 jobs:
-  build-ci-container:
+  # TODO(boomanaiden154): Switch this back to a single stage build when we can
+  # run this on the self-hosted runners and don't have to do it this way to
+  # avoid timeouts.
+  build-ci-container-stage1:
     if: github.repository_owner == 'llvm'
     runs-on: ubuntu-latest
+    steps:
+      - name: Checkout LLVM
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/workflows/containers/github-action-ci/
+      - name: Change podman Root Direcotry
+        run: |
+          mkdir -p ~/.config/containers
+          sudo mkdir -p /mnt/podman
+          sudo chown `whoami`:`whoami` /mnt/podman
+          cp ./.github/workflows/containers/github-action-ci/storage.conf ~/.config/containers/storage.conf
+          podman info
+      - name: Build container stage1
+        working-directory: ./.github/workflows/containers/github-action-ci/
+        run: |
+          podman build -t stage1-toolchain --target stage1-toolchain -f stage1.Dockerfile .
+      - name: Save container image
+        run: |
+          podman save stage1-toolchain > stage1-toolchain.tar
+      - name: Upload container image
+        uses: actions/upload-artifact@v4
+        with:
+          name: stage1-toolchain
+          path: stage1-toolchain.tar
+          retention-days: 1
+  build-ci-container-stage2:
+    if: github.repository_owner == 'llvm'
+    runs-on: ubuntu-latest
+    needs: build-ci-container-stage1
     permissions:
       packages: write
     steps:
@@ -38,10 +69,27 @@ jobs:
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci/
 
+      - name: Change podman Root Direcotry
+        run: |
+          mkdir -p ~/.config/containers
+          sudo mkdir -p /mnt/podman
+          sudo chown `whoami`:`whoami` /mnt/podman
+          cp ./.github/workflows/containers/github-action-ci/storage.conf ~/.config/containers/storage.conf
+          podman info
+
+      - name: Download stage1-toolchain
+        uses: actions/download-artifact@v4
+        with:
+          name: stage1-toolchain
+
+      - name: Load stage1-toolchain
+        run: |
+          podman load -i stage1-toolchain.tar
+
       - name: Build Container
         working-directory: ./.github/workflows/containers/github-action-ci/
         run: |
-          podman build -t ${{ steps.vars.outputs.container-name-tag }} .
+          podman build -t ${{ steps.vars.outputs.container-name-tag }} -f stage2.Dockerfile .
           podman tag ${{ steps.vars.outputs.container-name-tag }} ${{ steps.vars.outputs.container-name }}:latest
 
       - name: Test Container

.github/workflows/containers/github-action-ci/bootstrap.patch

@@ -0,0 +1,13 @@
+diff --git a/clang/cmake/caches/BOLT-PGO.cmake b/clang/cmake/caches/BOLT-PGO.cmake
+index 1a04ca9a74e5..d092820e4115 100644
+--- a/clang/cmake/caches/BOLT-PGO.cmake
++++ b/clang/cmake/caches/BOLT-PGO.cmake
+@@ -4,6 +4,8 @@ set(CLANG_BOOTSTRAP_TARGETS
+   stage2-clang-bolt
+   stage2-distribution
+   stage2-install-distribution
++  clang
++  lld
+   CACHE STRING "")
+ set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS
+   clang-bolt

.github/workflows/containers/github-action-ci/stage1.Dockerfile

@@ -0,0 +1,44 @@
+FROM docker.io/library/ubuntu:22.04 as base
+ENV LLVM_SYSROOT=/opt/llvm
+
+FROM base as stage1-toolchain
+ENV LLVM_VERSION=17.0.6
+
+RUN apt-get update && \
+    apt-get install -y \
+    wget \
+    gcc \
+    g++ \
+    cmake \
+    ninja-build \
+    python3 \
+    git \
+    curl
+
+RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$LLVM_VERSION.tar.gz && tar -xf llvmorg-$LLVM_VERSION.tar.gz
+
+WORKDIR /llvm-project-llvmorg-$LLVM_VERSION
+
+COPY bootstrap.patch /
+
+# TODO(boomanaiden154): Remove the patch pulled from a LLVM PR once we bump
+# the toolchain to version 18 and the patch is in-tree.
+# TODO(boomanaiden154): Remove the bootstrap patch once we unsplit the build
+# and no longer need to explicitly build the stage2 dependencies.
+RUN curl https://github.com/llvm/llvm-project/commit/dd0356d741aefa25ece973d6cc4b55dcb73b84b4.patch | patch -p1 && cat /bootstrap.patch | patch -p1
+
+RUN mkdir build
+
+RUN cmake -B ./build -G Ninja ./llvm \
+  -C ./clang/cmake/caches/BOLT-PGO.cmake \
+  -DBOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DBOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DPGO_INSTRUMENT_LTO=Thin \
+  -DLLVM_ENABLE_RUNTIMES="compiler-rt" \
+  -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
+  -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
+  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format" \
+  -DCLANG_DEFAULT_LINKER="lld" \
+  -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=/llvm-project-llvmorg-$LLVM_VERSION/llvm
+
+RUN ninja -C ./build stage2-instrumented-clang stage2-instrumented-lld

.github/workflows/containers/github-action-ci/stage2.Dockerfile

@@ -0,0 +1,27 @@
+FROM docker.io/library/ubuntu:22.04 as base
+ENV LLVM_SYSROOT=/opt/llvm
+
+FROM stage1-toolchain AS stage2-toolchain
+
+RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution && rm -rf ./build
+
+FROM base
+
+COPY --from=stage2-toolchain $LLVM_SYSROOT $LLVM_SYSROOT
+
+# Need to install curl for hendrikmuhs/ccache-action
+# Need nodejs for some of the GitHub actions.
+# Need perl-modules for clang analyzer tests.
+RUN apt-get update && \
+    apt-get install -y \
+    binutils \
+    cmake \
+    curl \
+    libstdc++-11-dev \
+    ninja-build \
+    nodejs \
+    perl-modules \
+    python3-psutil
+
+ENV LLVM_SYSROOT=$LLVM_SYSROOT
+ENV PATH=${LLVM_SYSROOT}/bin:${PATH}

.github/workflows/containers/github-action-ci/storage.conf

@@ -0,0 +1,4 @@
+[storage]
+  driver = "overlay"
+  runroot = "/mnt/podman/container"
+  graphroot = "/mnt/podman/image"

.github/workflows/email-check.yaml

@@ -0,0 +1,44 @@
+name: "Check for private emails used in PRs"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+
+permissions:
+  contents: read
+
+jobs:
+  validate_email:
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-latest
+    if: github.repository == 'llvm/llvm-project'
+    steps:
+      - name: Fetch LLVM sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Extract author email
+        id: author
+        run: |
+          git log -1
+          echo "EMAIL=$(git show -s --format='%ae' HEAD~0)" >> $GITHUB_OUTPUT
+
+      - name: Validate author email
+        if: ${{ endsWith(steps.author.outputs.EMAIL, 'noreply.github.com')  }}
+        uses: actions/github-script@v6
+        env:
+          EMAIL: ${{ steps.author.outputs.EMAIL }}
+        with:
+          script: |
+            const { EMAIL } = process.env
+            await github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `⚠️ We detected that you are using a GitHub private e-mail address to contribute to the repo.
+              Please turn off [Keep my email addresses private](https://github.com/settings/emails) setting in your account.
+              See [LLVM Discourse](https://discourse.llvm.org/t/hidden-emails-on-github-should-we-do-something-about-it) for more information.
+            `})

.github/workflows/libclc-tests.yml

@@ -36,5 +36,4 @@ jobs:
     name: Test libclc
     uses: ./.github/workflows/llvm-project-tests.yml
     with:
-      build_target: ''
       projects: clang;libclc

.github/workflows/lldb-tests.yml

@@ -36,5 +36,4 @@ jobs:
     name: Build lldb
     uses: ./.github/workflows/llvm-project-tests.yml
     with:
-      build_target: ''
       projects: clang;lldb

.github/workflows/llvm-project-tests.yml

@@ -22,8 +22,9 @@ on:
   workflow_call:
     inputs:
       build_target:
-        required: true
+        required: false
         type: string
+        default: "all"
 
       projects:
         required: true
@@ -98,20 +99,40 @@ jobs:
           key: ${{ matrix.os }}
           variant: sccache
       - name: Build and Test
-        uses: llvm/actions/build-test-llvm-project@main
         env:
           # Workaround for https://github.com/actions/virtual-environments/issues/5900.
           # This should be a no-op for non-mac OSes
           PKG_CONFIG_PATH: /usr/local/Homebrew/Library/Homebrew/os/mac/pkgconfig//12
-        with:
-          cmake_args: '-GNinja -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLDB_INCLUDE_TESTS=OFF -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ${{ inputs.extra_cmake_args }}'
-          build_target: '${{ inputs.build_target }}'
+        shell: bash
+        id: build-llvm
+        run: |
+          if [ "${{ runner.os }}" == "Linux" ]; then
+            builddir="/mnt/build/"
+            sudo mkdir -p $builddir
+            sudo chown `whoami`:`whoami` $builddir
+          else
+            builddir="$(pwd)"/build
+          fi
+          echo "llvm-builddir=$builddir" >> "$GITHUB_OUTPUT"
+          cmake -G Ninja \
+                -B "$builddir" \
+                -S llvm \
+                -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DLLVM_ENABLE_ASSERTIONS=ON \
+                -DLLDB_INCLUDE_TESTS=OFF \
+                -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+                -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+                ${{ inputs.extra_cmake_args }}
+          ninja -C "$builddir" '${{ inputs.build_target }}'
 
       - name: Build and Test libclc
         if: "!startsWith(matrix.os, 'windows') && contains(inputs.projects, 'libclc')"
+        env:
+          LLVM_BUILDDIR: ${{ steps.build-llvm.outputs.llvm-builddir }}
         run: |
           # Make sure all of LLVM libraries that llvm-config needs are built.
-          ninja -C build
-          cmake -G Ninja -S libclc -B libclc-build -DLLVM_DIR="$(pwd)"/build/lib/cmake/llvm -DLIBCLC_TARGETS_TO_BUILD="amdgcn--;amdgcn--amdhsa;r600--;nvptx--;nvptx64--;nvptx--nvidiacl;nvptx64--nvidiacl"
+          ninja -C "$LLVM_BUILDDIR"
+          cmake -G Ninja -S libclc -B libclc-build -DLLVM_DIR="$LLVM_BUILDDIR"/lib/cmake/llvm -DLIBCLC_TARGETS_TO_BUILD="amdgcn--;amdgcn--amdhsa;r600--;nvptx--;nvptx64--;nvptx--nvidiacl;nvptx64--nvidiacl"
           ninja -C libclc-build
           ninja -C libclc-build test

.github/workflows/pr-code-format.yml

@@ -1,5 +1,9 @@
 name: "Check code formatting"
-on: pull_request_target
+on:
+  pull_request_target:
+    branches:
+      - main
+
 permissions:
   pull-requests: write
 

.github/workflows/spirv-tests.yml

@@ -25,5 +25,5 @@ jobs:
     with:
       build_target: check-llvm-codegen-spirv
       projects:
-      extra_cmake_args: '-DLLVM_TARGETS_TO_BUILD="" -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="SPIRV"'
+      extra_cmake_args: '-DLLVM_TARGETS_TO_BUILD="" -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="SPIRV" -DLLVM_INCLUDE_SPIRV_TOOLS_TESTS=ON'
       os_list: '["ubuntu-latest"]'

bolt/include/bolt/Core/BinaryContext.h

@@ -671,6 +671,11 @@ class BinaryContext {
     uint64_t StaleSampleCount{0};
     ///   the count of matched samples
     uint64_t MatchedSampleCount{0};
+    ///   the number of stale functions that have matching number of blocks in
+    ///   the profile
+    uint64_t NumStaleFuncsWithEqualBlockCount{0};
+    ///   the number of blocks that have matching size but a differing hash
+    uint64_t NumStaleBlocksWithEqualIcount{0};
   } Stats;
 
   // Address of the first allocated segment.

bolt/include/bolt/Rewrite/RewriteInstance.h

@@ -264,6 +264,11 @@ class RewriteInstance {
   void createPLTBinaryFunction(uint64_t TargetAddress, uint64_t EntryAddress,
                                uint64_t EntrySize);
 
+  /// Disassemble PLT instruction.
+  void disassemblePLTInstruction(const BinarySection &Section,
+                                 uint64_t InstrOffset, MCInst &Instruction,
+                                 uint64_t &InstrSize);
+
   /// Disassemble aarch64-specific .plt \p Section auxiliary function
   void disassemblePLTSectionAArch64(BinarySection &Section);
 

bolt/lib/Core/BinarySection.cpp

@@ -72,7 +72,8 @@ BinarySection::hash(const BinaryData &BD,
 
 void BinarySection::emitAsData(MCStreamer &Streamer,
                                const Twine &SectionName) const {
-  StringRef SectionContents = getContents();
+  StringRef SectionContents =
+      isFinalized() ? getOutputContents() : getContents();
   MCSectionELF *ELFSection =
       BC.Ctx->getELFSection(SectionName, getELFType(), getELFFlags());