Skip to content

Commit

Permalink
[FLINK-35577] Fix the broken CI
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zakelly @mayuehappy
Zakelly authored and mayuehappy committed Jun 24, 2024
1 parent 96705d2 commit 09f9d06
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 52 deletions.
119 changes: 68 additions & 51 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,17 @@ commands:
sudo apt remove --purge cmake
sudo snap install cmake --classic
install-clang-10:
steps:
- run:
name: Install Clang 10
command: |
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list
echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
sudo apt-get update -y && sudo apt-get install -y clang-10
install-gflags:
steps:
- run:
Expand Down Expand Up @@ -237,7 +248,7 @@ jobs:
build-macos:
macos:
xcode: 14.3.1
resource_class: medium
resource_class: macos.m1.medium.gen1
environment:
ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now
steps:
Expand Down Expand Up @@ -289,15 +300,15 @@ jobs:
resource_class: xlarge
steps:
- pre-steps
- run: make V=1 J=32 -j32 check
- run: make V=1 J=8 -j8 check
- post-steps

build-linux-encrypted_env-no_compression:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
- run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=8 -j8 check
- run: |
./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression
- post-steps
Expand All @@ -307,28 +318,28 @@ jobs:
resource_class: xlarge
steps:
- pre-steps
- run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
- run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j8 check
- post-steps

build-linux-release:
executor: linux-docker
resource_class: xlarge
steps:
- checkout # check out the code in the project directory
- run: make V=1 -j32 LIB_MODE=shared release
- run: make V=1 -j8 LIB_MODE=shared release
- run: ls librocksdb.so # ensure shared lib built
- run: ./db_stress --version # ensure with gflags
- run: make clean
- run: make V=1 -j32 release
- run: make V=1 -j8 release
- run: ls librocksdb.a # ensure static lib built
- run: ./db_stress --version # ensure with gflags
- run: make clean
- run: apt-get remove -y libgflags-dev
- run: make V=1 -j32 LIB_MODE=shared release
- run: make V=1 -j8 LIB_MODE=shared release
- run: ls librocksdb.so # ensure shared lib built
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
- run: make clean
- run: make V=1 -j32 release
- run: make V=1 -j8 release
- run: ls librocksdb.a # ensure static lib built
- run: if ./db_stress --version; then false; else true; fi # ensure without gflags
- post-steps
Expand All @@ -354,43 +365,49 @@ jobs:
- post-steps

build-linux-clang10-asan:
executor: linux-docker
machine:
image: ubuntu-2004:202111-02
resource_class: xlarge
steps:
- pre-steps
- install-gflags
- install-clang-10
- run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out
- post-steps

build-linux-clang10-mini-tsan:
executor: linux-docker
resource_class: xlarge+
machine:
image: ubuntu-2004:202111-02
resource_class: xlarge
steps:
- pre-steps
- run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
- install-gflags
- install-clang-10
- run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out.
- post-steps

build-linux-clang10-ubsan:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out
- run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j8 ubsan_check # aligned new doesn't work for reason we haven't figured out
- post-steps

build-linux-valgrind:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: PORTABLE=1 make V=1 -j32 valgrind_test
- run: PORTABLE=1 make V=1 -j8 valgrind_test
- post-steps

build-linux-clang10-clang-analyze:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
- run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j8 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
- post-steps
- run:
name: "compress test report"
Expand Down Expand Up @@ -420,7 +437,7 @@ jobs:
- pre-steps
- setup-folly
- build-folly
- run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
- run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j8 && ctest -j8)
- post-steps

build-linux-cmake-with-folly-lite-no-test:
Expand All @@ -429,15 +446,15 @@ jobs:
steps:
- pre-steps
- setup-folly
- run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)
- run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j8)
- post-steps

build-linux-cmake-with-benchmark:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20
- run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j8 && ctest -j8
- post-steps

build-linux-unity-and-headers:
Expand All @@ -463,7 +480,7 @@ jobs:
- pre-steps
- setup-folly
- build-folly
- run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures
- run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures
- post-steps

build-linux-gcc-7-with-folly-lite-no-test:
Expand All @@ -472,15 +489,15 @@ jobs:
steps:
- pre-steps
- setup-folly
- run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all
- run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j8 all
- post-steps

build-linux-gcc-8-no_test_run:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all
- run: CC=gcc-8 CXX=g++-8 V=1 make -j8 all
- post-steps

build-linux-cmake-with-folly-coroutines:
Expand All @@ -493,31 +510,31 @@ jobs:
- pre-steps
- setup-folly
- build-folly
- run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)
- run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j8 && ctest -j8)
- post-steps

build-linux-gcc-10-cxx20-no_test_run:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
- run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j8 all
- post-steps

build-linux-gcc-11-no_test_run:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench # TODO: LIB_MODE only to work around unresolved linker failures
- run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j8 all microbench # TODO: LIB_MODE only to work around unresolved linker failures
- post-steps

build-linux-clang-13-no_test_run:
executor: linux-docker
resource_class: xlarge
steps:
- pre-steps
- run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
- run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j8 all microbench
- post-steps

# Ensure ASAN+UBSAN with folly, and full testsuite with clang 13
Expand All @@ -528,7 +545,7 @@ jobs:
- pre-steps
- setup-folly
- build-folly
- run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures
- run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures
- post-steps

# This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
Expand All @@ -537,7 +554,7 @@ jobs:
resource_class: xlarge
steps:
- pre-steps
- run: DEBUG_LEVEL=0 make -j32 run_microbench
- run: DEBUG_LEVEL=0 make -j8 run_microbench
- post-steps

build-linux-mini-crashtest:
Expand All @@ -555,7 +572,7 @@ jobs:
- pre-steps
- run:
name: "run crashtest"
command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage
command: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage
no_output_timeout: 100m
- post-steps

Expand All @@ -566,7 +583,7 @@ jobs:
- pre-steps
- run:
name: "run crashtest"
command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage
command: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage
no_output_timeout: 100m
- post-steps

Expand Down Expand Up @@ -641,26 +658,26 @@ jobs:
command: make V=1 J=8 -j8 jtest
- post-steps

build-linux-java-pmd:
machine:
image: ubuntu-2004:202111-02
resource_class: large
environment:
JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
steps:
- install-maven
- pre-steps
- run:
name: "Set Java Environment"
command: |
echo "JAVA_HOME=${JAVA_HOME}"
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run:
name: "PMD RocksDBJava"
command: make V=1 J=8 -j8 jpmd
- post-pmd-steps
# build-linux-java-pmd:
# machine:
# image: ubuntu-2004:202111-02
# resource_class: large
# environment:
# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
# steps:
# - install-maven
# - pre-steps
# - run:
# name: "Set Java Environment"
# command: |
# echo "JAVA_HOME=${JAVA_HOME}"
# echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
# which java && java -version
# which javac && javac -version
# - run:
# name: "PMD RocksDBJava"
# command: make V=1 J=8 -j8 jpmd
# - post-pmd-steps

build-linux-java-static:
executor: linux-java-docker
Expand Down Expand Up @@ -788,7 +805,7 @@ jobs:
TEST_TMPDIR: /tmp/rocksdb_test_tmp
steps:
- pre-steps
- run: make V=1 -j32 check
- run: make V=1 -j8 check
- post-steps

build-linux-arm-test-full:
Expand Down Expand Up @@ -925,7 +942,7 @@ workflows:
- build-macos-java
- build-macos-java-static
- build-macos-java-static-universal
- build-linux-java-pmd
# - build-linux-java-pmd
jobs-macos:
jobs:
- build-macos
Expand Down
3 changes: 2 additions & 1 deletion java/rocksjni/flink_compactionfilterjni.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@ class JniCallbackBase : public ROCKSDB_NAMESPACE::JniCallback {
protected:
inline void CheckAndRethrowException(JNIEnv* env) const {
if (env->ExceptionCheck()) {
jthrowable obj = env->ExceptionOccurred();
env->ExceptionDescribe();
env->Throw(env->ExceptionOccurred());
env->Throw(obj);
}
}
};
Expand Down

0 comments on commit 09f9d06

Please sign in to comment.