Merge branch 'main' into blas-opt

.github/workflows/bcload.yml

@@ -10,7 +10,8 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- llvm: ["11", "12", "13", "14", "15", "16"]
+ # BCLoad tests only run with old pass manager [aka up to and including 13]
+ llvm: ["11", "12", "13"]
  build: ["Release"] # "RelWithDebInfo"
  os: [ubuntu-20.04]
 
@@ -20,8 +21,8 @@ jobs:
  run: |
  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
  sudo apt-add-repository "deb http://apt.llvm.org/`lsb_release -c | cut -f2`/ llvm-toolchain-`lsb_release -c | cut -f2`-${{ matrix.llvm }} main" || true
- sudo apt-get install -y autoconf cmake gcc g++ libtool gfortran llvm-${{ matrix.llvm }}-dev libomp-${{ matrix.llvm }}-dev clang-${{ matrix.llvm }} libclang-${{ matrix.llvm }}-dev libeigen3-dev libboost-dev
- sudo python3 -m pip install --upgrade pip setuptools lit
+ sudo apt-get install -y cmake gcc g++ llvm-${{ matrix.llvm }}-dev clang-${{ matrix.llvm }} libclang-${{ matrix.llvm }}-dev
+ sudo python3 -m pip install --upgrade pip lit
  if [[ '${{ matrix.llvm }}' == '13' ]]; then
  sudo sed -i 's/add_executable(llvm-omp-device-info IMPORTED)//g' /usr/lib/llvm-${{matrix.llvm}}/lib/cmake/llvm/LLVMExports*.cmake
  sudo sed -i 's/llvm-omp-device-info//g' /usr/lib/llvm-${{matrix.llvm}}/lib/cmake/llvm/LLVMExports*.cmake

.github/workflows/ccpp.yml

@@ -28,8 +28,8 @@ jobs:
  run: |
  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
  sudo apt-add-repository "deb http://apt.llvm.org/`lsb_release -c | cut -f2`/ llvm-toolchain-`lsb_release -c | cut -f2`-${{ matrix.llvm }} main" || true
- sudo apt-get install -y cmake gcc g++ gfortran llvm-${{ matrix.llvm }}-dev libomp-${{ matrix.llvm }}-dev lld-${{ matrix.llvm }} clang-${{ matrix.llvm }} libclang-${{ matrix.llvm }}-dev libeigen3-dev libboost-dev
- sudo python3 -m pip install --upgrade pip setuptools lit
+ sudo apt-get install -y cmake gcc g++ llvm-${{ matrix.llvm }}-dev libomp-${{ matrix.llvm }}-dev lld-${{ matrix.llvm }} clang-${{ matrix.llvm }} libclang-${{ matrix.llvm }}-dev libeigen3-dev libboost-dev
+ sudo python3 -m pip install --upgrade pip lit
  if [[ '${{ matrix.llvm }}' == '13' ]]; then
  sudo sed -i 's/add_executable(llvm-omp-device-info IMPORTED)//g' /usr/lib/llvm-${{matrix.llvm}}/lib/cmake/llvm/LLVMExports*.cmake
  sudo sed -i 's/llvm-omp-device-info//g' /usr/lib/llvm-${{matrix.llvm}}/lib/cmake/llvm/LLVMExports*.cmake

.github/workflows/doxygen.yml

@@ -4,7 +4,6 @@ on:
  push:
  branches:
  - main
- merge_group:
 
 jobs:
  docs:

.github/workflows/enzyme-bazel.yml

@@ -29,7 +29,7 @@ jobs:
  run: |
  sudo rm -f /etc/apt/sources.list.d/*llvm*.list
  sudo apt-get update
- sudo apt-get install -y ninja-build git autoconf cmake gcc g++ libtool gfortran python3 python3-dev 
+ sudo apt-get install -y ninja-build git autoconf cmake gcc g++ libtool python3 python3-dev 
  - uses: actions/checkout@v3
  - uses: actions/checkout@v3
  with:

.github/workflows/enzyme-ci.yml

@@ -26,8 +26,8 @@ jobs:
  run: |
  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
  sudo apt-add-repository "deb http://apt.llvm.org/`lsb_release -c | cut -f2`/ llvm-toolchain-`lsb_release -c | cut -f2`-${{ matrix.llvm }} main" || true
- sudo apt-get install -y cmake gcc g++ gfortran llvm-${{ matrix.llvm }}-dev
- sudo python3 -m pip install --upgrade pip setuptools lit
+ sudo apt-get install -y cmake gcc g++ llvm-${{ matrix.llvm }}-dev
+ sudo python3 -m pip install --upgrade pip lit
  if [[ '${{ matrix.llvm }}' == '13' ]]; then
  sudo sed -i 's/add_executable(llvm-omp-device-info IMPORTED)//g' /usr/lib/llvm-${{matrix.llvm}}/lib/cmake/llvm/LLVMExports*.cmake
  sudo sed -i 's/llvm-omp-device-info//g' /usr/lib/llvm-${{matrix.llvm}}/lib/cmake/llvm/LLVMExports*.cmake
@@ -61,7 +61,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- llvm: ["9", "11", "12", "13", "14", "15"]
+ llvm: ["11", "12", "13", "14", "15"]
  build: ["Release", "Debug"] # "RelWithDebInfo"
 
  timeout-minutes: 30 
@@ -70,7 +70,7 @@ jobs:
  run: |
  brew update
  brew install llvm@${{ matrix.llvm }} make cmake
- sudo python3 -m pip install --upgrade pip setuptools lit requests
+ sudo python3 -m pip install --upgrade pip lit requests
  - uses: actions/checkout@v3
  - name: mkdir
  run: rm -rf build && mkdir build
@@ -108,7 +108,7 @@ jobs:
  - name: add llvm
  run: |
  brew install llvm@${{ matrix.llvm }} make cmake gcc
- sudo python3 -m pip install --upgrade pip setuptools lit
+ sudo python3 -m pip install --upgrade pip lit
  - uses: actions/checkout@v3
  - name: mkdir
  run: rm -rf build && mkdir build

.github/workflows/fortran.yml

@@ -29,8 +29,8 @@ jobs:
  run: |
  wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
  sudo apt-add-repository "deb http://apt.llvm.org/`lsb_release -c | cut -f2`/ llvm-toolchain-`lsb_release -c | cut -f2`-${{ matrix.llvm }} main" || true
- sudo apt-get update && sudo apt-get install -y cmake gcc g++ gfortran llvm-${{ matrix.llvm }}-dev ninja-build pip
- sudo python3 -m pip install --upgrade pip setuptools lit
+ sudo apt-get update && sudo apt-get install -y cmake gcc g++ llvm-${{ matrix.llvm }}-dev ninja-build pip
+ sudo python3 -m pip install --upgrade pip lit
  - name: add intel tools
  run: |
  wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
@@ -51,4 +51,4 @@ jobs:
  - name: run Fortran tests
  working-directory: 'build'
  run: ninja check-enzyme-fortran
-
+

enzyme/CMakeLists.txt

@@ -30,6 +30,7 @@ option(ENZYME_FLANG "Build enzyme flang symlink" OFF)
 option(ENZYME_MLIR "Build enzyme mlir plugin" OFF)
 option(ENZYME_IFX "Enable enzyme support for the Intel Fortran compiler IFX" OFF)
 option(ENZYME_EXTERNAL_SHARED_LIB "Build external shared library" OFF)
+option(ENZYME_STATIC_LIB "Build static library" OFF)
 set(ENZYME_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(ENZYME_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 list(APPEND CMAKE_MODULE_PATH "${ENZYME_SOURCE_DIR}/cmake/modules")

enzyme/Enzyme/ActivityAnalysis.cpp

@@ -63,6 +63,13 @@
 
 using namespace llvm;
 
+#if LLVM_VERSION_MAJOR >= 14
+#define addAttribute addAttributeAtIndex
+#define removeAttribute removeAttributeAtIndex
+#define getAttribute getAttributeAtIndex
+#define hasAttribute hasAttributeAtIndex
+#endif
+
 extern "C" {
 cl::opt<bool>
  EnzymePrintActivity("enzyme-print-activity", cl::init(false), cl::Hidden,
@@ -484,27 +491,6 @@ bool ActivityAnalyzer::isFunctionArgumentConstant(CallInst *CI, Value *val) {
  CI->getArgOperand(0) != val && CI->getArgOperand(1) != val)
  return true;
 
- // only the float arg input is potentially active
- if (Name == "frexp" || Name == "frexpf" || Name == "frexpl") {
- return val != CI->getOperand(0);
- }
-
- // The relerr argument is inactive
- if (Name == "Faddeeva_erf" || Name == "Faddeeva_erfc" ||
- Name == "Faddeeva_erfcx" || Name == "Faddeeva_erfi" ||
- Name == "Faddeeva_dawson") {
-#if LLVM_VERSION_MAJOR >= 14
- for (size_t i = 0; i < CI->arg_size() - 1; i++)
-#else
- for (size_t i = 0; i < CI->getNumArgOperands() - 1; i++)
-#endif
- {
- if (val == CI->getOperand(i))
- return false;
- }
- return true;
- }
-
  // only the buffer is active for mpi send/recv
  if (Name == "MPI_Recv" || Name == "PMPI_Recv" || Name == "MPI_Send" ||
  Name == "PMPI_Send") {
@@ -550,23 +536,6 @@ static inline void propagateArgumentInformation(
  propagateFromOperand(CI.getArgOperand(0));
  return;
  }
- if (Name == "frexp" || Name == "frexpf" || Name == "frexpl") {
- propagateFromOperand(CI.getOperand(0));
- return;
- }
- if (Name == "Faddeeva_erf" || Name == "Faddeeva_erfc" ||
- Name == "Faddeeva_erfcx" || Name == "Faddeeva_erfi" ||
- Name == "Faddeeva_dawson") {
-#if LLVM_VERSION_MAJOR >= 14
- for (size_t i = 0; i < CI.arg_size() - 1; i++)
-#else
- for (size_t i = 0; i < CI.getNumArgOperands() - 1; i++)
-#endif
- {
- propagateFromOperand(CI.getOperand(i));
- }
- return;
- }
 
  if (Name == "julia.call" || Name == "julia.call2") {
 #if LLVM_VERSION_MAJOR >= 14
@@ -1236,15 +1205,35 @@ bool ActivityAnalyzer::isConstantValue(TypeResults const &TR, Value *Val) {
  return false;
  }
 
+ if (auto I = dyn_cast<Instruction>(Val)) {
+ if (hasMetadata(I, "enzyme_active") ||
+ hasMetadata(I, "enzyme_active_val")) {
+ if (EnzymePrintActivity)
+ llvm::errs() << "forced active val (MD)" << *Val << "\n";
+ InsertConstantValue(TR, Val);
+ return true;
+ }
+ if (hasMetadata(I, "enzyme_inactive") ||
+ hasMetadata(I, "enzyme_inactive_val")) {
+ if (EnzymePrintActivity)
+ llvm::errs() << "forced inactive val (MD)" << *Val << "\n";
+ InsertConstantValue(TR, Val);
+ return true;
+ }
+ }
  if (auto CI = dyn_cast<CallInst>(Val)) {
- if (CI->hasFnAttr("enzyme_active") || CI->hasFnAttr("enzyme_active_val")) {
+ if (CI->hasFnAttr("enzyme_active") || CI->hasFnAttr("enzyme_active_val") ||
+ CI->getAttributes().hasAttribute(llvm::AttributeList::ReturnIndex,
+ "enzyme_active")) {
  if (EnzymePrintActivity)
  llvm::errs() << "forced active val " << *Val << "\n";
  ActiveValues.insert(Val);
  return false;
  }
  if (CI->hasFnAttr("enzyme_inactive") ||
- CI->hasFnAttr("enzyme_inactive_val")) {
+ CI->hasFnAttr("enzyme_inactive_val") ||
+ CI->getAttributes().hasAttribute(llvm::AttributeList::ReturnIndex,
+ "enzyme_inactive")) {
  if (EnzymePrintActivity)
  llvm::errs() << "forced inactive val " << *Val << "\n";
  InsertConstantValue(TR, Val);
@@ -1254,14 +1243,18 @@ bool ActivityAnalyzer::isConstantValue(TypeResults const &TR, Value *Val) {
 
  if (called) {
  if (called->hasFnAttribute("enzyme_active") ||
- called->hasFnAttribute("enzyme_active_val")) {
+ called->hasFnAttribute("enzyme_active_val") ||
+ called->getAttributes().hasAttribute(llvm::AttributeList::ReturnIndex,
+ "enzyme_active")) {
  if (EnzymePrintActivity)
  llvm::errs() << "forced active val " << *Val << "\n";
  ActiveValues.insert(Val);
  return false;
  }
  if (called->hasFnAttribute("enzyme_inactive") ||
- called->hasFnAttribute("enzyme_inactive_val")) {
+ called->hasFnAttribute("enzyme_inactive_val") ||
+ called->getAttributes().hasAttribute(llvm::AttributeList::ReturnIndex,
+ "enzyme_inactive")) {
  if (EnzymePrintActivity)
  llvm::errs() << "forced inactive val " << *Val << "\n";
  InsertConstantValue(TR, Val);
@@ -1422,7 +1415,9 @@ bool ActivityAnalyzer::isConstantValue(TypeResults const &TR, Value *Val) {
  }
  } else if (auto op = dyn_cast<CallInst>(TmpOrig)) {
  if (op->hasFnAttr("enzyme_inactive") ||
- op->hasFnAttr("enzyme_inactive_val")) {
+ op->hasFnAttr("enzyme_inactive_val") ||
+ op->getAttributes().hasAttribute(llvm::AttributeList::ReturnIndex,
+ "enzyme_inactive")) {
  InsertConstantValue(TR, Val);
  insertConstantsFrom(TR, *UpHypothesis);
  return true;
@@ -1431,8 +1426,11 @@ bool ActivityAnalyzer::isConstantValue(TypeResults const &TR, Value *Val) {
 
  StringRef funcName = getFuncNameFromCall(op);
 
- if (called && (called->hasFnAttribute("enzyme_inactive") ||
- called->hasFnAttribute("enzyme_inactive_val"))) {
+ if (called &&
+ (called->hasFnAttribute("enzyme_inactive") ||
+ called->hasFnAttribute("enzyme_inactive_val") ||
+ called->getAttributes().hasAttribute(
+ llvm::AttributeList::ReturnIndex, "enzyme_inactive"))) {
  InsertConstantValue(TR, Val);
  insertConstantsFrom(TR, *UpHypothesis);
  return true;
@@ -1536,6 +1534,7 @@ bool ActivityAnalyzer::isConstantValue(TypeResults const &TR, Value *Val) {
  if (directions & DOWN &&
  (funcName == "malloc" || funcName == "calloc" ||
  funcName == "_Znwm" || funcName == "julia.gc_alloc_obj" ||
+ funcName == "??2@YAPAXI@Z" || funcName == "??2@YAPEAX_K@Z" ||
  funcName == "jl_gc_alloc_typed" ||
  funcName == "ijl_gc_alloc_typed")) {
  std::shared_ptr<ActivityAnalyzer> Hypothesis =
@@ -2929,7 +2928,8 @@ bool ActivityAnalyzer::isValueInactiveFromUsers(TypeResults const &TR,
 
  if (F) {
  if (UA == UseActivity::AllStores &&
- F->getName() == "julia.write_barrier")
+ (F->getName() == "julia.write_barrier" ||
+ F->getName() == "julia.write_barrier_binding"))
  continue;
  if (F->getIntrinsicID() == Intrinsic::memcpy ||
  F->getIntrinsicID() == Intrinsic::memmove) {
@@ -3082,11 +3082,19 @@ bool ActivityAnalyzer::isValueInactiveFromUsers(TypeResults const &TR,
  }
  if (legal) {
  toContinue = true;
- break;
  }
  }
- if (toContinue)
+ if (toContinue) {
+ if (EnzymePrintActivity) {
+ llvm::errs() << "Value found indirect call use which must be "
+ "constant as all stored functions are constant val:"
+ << *val << " user " << *call << "\n";
+ }
+ for (auto u : call->users()) {
+ todo.push_back(std::make_tuple(u, a, UseActivity::None));
+ }
  continue;
+ }
  }
  }
 

enzyme/Enzyme/ActivityAnalysisPrinter.cpp

@@ -89,7 +89,7 @@ bool printActivityAnalysis(llvm::Function &F, TargetLibraryInfo &TLI) {
  if (a.getType()->isFPOrFPVectorTy()) {
  dt = ConcreteType(a.getType()->getScalarType());
  } else if (a.getType()->isPointerTy()) {
-#if LLVM_VERSION_MAJOR >= 18
+#if LLVM_VERSION_MAJOR >= 17
 #else
  auto et = a.getType()->getPointerElementType();
  if (et->isFPOrFPVectorTy()) {
@@ -113,7 +113,7 @@ bool printActivityAnalysis(llvm::Function &F, TargetLibraryInfo &TLI) {
  if (F.getReturnType()->isFPOrFPVectorTy()) {
  dt = ConcreteType(F.getReturnType()->getScalarType());
  } else if (F.getReturnType()->isPointerTy()) {
-#if LLVM_VERSION_MAJOR >= 18
+#if LLVM_VERSION_MAJOR >= 17
 #else
  auto et = F.getReturnType()->getPointerElementType();
  if (et->isFPOrFPVectorTy()) {