From 09f9d06520ff7a47aeccd443b7b03040cd57d2d9 Mon Sep 17 00:00:00 2001 From: Zakelly Date: Mon, 18 Jul 2022 21:50:50 +0800 Subject: [PATCH] [FLINK-35577] Fix the broken CI --- .circleci/config.yml | 119 ++++++++++++--------- java/rocksjni/flink_compactionfilterjni.cc | 3 +- 2 files changed, 70 insertions(+), 52 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d4f15512..84b39239a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,17 @@ commands: sudo apt remove --purge cmake sudo snap install cmake --classic + install-clang-10: + steps: + - run: + name: Install Clang 10 + command: | + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - + echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list + echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" | sudo tee -a /etc/apt/sources.list + echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable + sudo apt-get update -y && sudo apt-get install -y clang-10 + install-gflags: steps: - run: @@ -237,7 +248,7 @@ jobs: build-macos: macos: xcode: 14.3.1 - resource_class: medium + resource_class: macos.m1.medium.gen1 environment: ROCKSDB_DISABLE_JEMALLOC: 1 # jemalloc cause env_test hang, disable it for now steps: @@ -289,7 +300,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: make V=1 J=32 -j32 check + - run: make V=1 J=8 -j8 check - post-steps build-linux-encrypted_env-no_compression: @@ -297,7 +308,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check + - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=8 -j8 check - run: | ./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression - post-steps @@ -307,7 +318,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check + - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j8 check - post-steps build-linux-release: @@ -315,20 +326,20 @@ jobs: resource_class: xlarge steps: - checkout # check out the code in the project directory - - run: make V=1 -j32 LIB_MODE=shared release + - run: make V=1 -j8 LIB_MODE=shared release - run: ls librocksdb.so # ensure shared lib built - run: ./db_stress --version # ensure with gflags - run: make clean - - run: make V=1 -j32 release + - run: make V=1 -j8 release - run: ls librocksdb.a # ensure static lib built - run: ./db_stress --version # ensure with gflags - run: make clean - run: apt-get remove -y libgflags-dev - - run: make V=1 -j32 LIB_MODE=shared release + - run: make V=1 -j8 LIB_MODE=shared release - run: ls librocksdb.so # ensure shared lib built - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - run: make clean - - run: make V=1 -j32 release + - run: make V=1 -j8 release - run: ls librocksdb.a # ensure static lib built - run: if ./db_stress --version; then false; else true; fi # ensure without gflags - post-steps @@ -354,19 +365,25 @@ jobs: - post-steps build-linux-clang10-asan: - executor: linux-docker + machine: + image: ubuntu-2004:202111-02 resource_class: xlarge steps: - pre-steps + - install-gflags + - install-clang-10 - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out - post-steps build-linux-clang10-mini-tsan: - executor: linux-docker - resource_class: xlarge+ + machine: + image: ubuntu-2004:202111-02 + resource_class: xlarge steps: - pre-steps - - run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check + - install-gflags + - install-clang-10 + - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check # aligned new doesn't work for reason we haven't figured out. - post-steps build-linux-clang10-ubsan: @@ -374,7 +391,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check # aligned new doesn't work for reason we haven't figured out + - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j8 ubsan_check # aligned new doesn't work for reason we haven't figured out - post-steps build-linux-valgrind: @@ -382,7 +399,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: PORTABLE=1 make V=1 -j32 valgrind_test + - run: PORTABLE=1 make V=1 -j8 valgrind_test - post-steps build-linux-clang10-clang-analyze: @@ -390,7 +407,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. + - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j8 analyze # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path. - post-steps - run: name: "compress test report" @@ -420,7 +437,7 @@ jobs: - pre-steps - setup-folly - build-folly - - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - run: (mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j8 && ctest -j8) - post-steps build-linux-cmake-with-folly-lite-no-test: @@ -429,7 +446,7 @@ jobs: steps: - pre-steps - setup-folly - - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20) + - run: (mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j8) - post-steps build-linux-cmake-with-benchmark: @@ -437,7 +454,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 + - run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j8 && ctest -j8 - post-steps build-linux-unity-and-headers: @@ -463,7 +480,7 @@ jobs: - pre-steps - setup-folly - build-folly - - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures + - run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures - post-steps build-linux-gcc-7-with-folly-lite-no-test: @@ -472,7 +489,7 @@ jobs: steps: - pre-steps - setup-folly - - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all + - run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j8 all - post-steps build-linux-gcc-8-no_test_run: @@ -480,7 +497,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all + - run: CC=gcc-8 CXX=g++-8 V=1 make -j8 all - post-steps build-linux-cmake-with-folly-coroutines: @@ -493,7 +510,7 @@ jobs: - pre-steps - setup-folly - build-folly - - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20) + - run: (mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j8 && ctest -j8) - post-steps build-linux-gcc-10-cxx20-no_test_run: @@ -501,7 +518,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all + - run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j8 all - post-steps build-linux-gcc-11-no_test_run: @@ -509,7 +526,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench # TODO: LIB_MODE only to work around unresolved linker failures + - run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j8 all microbench # TODO: LIB_MODE only to work around unresolved linker failures - post-steps build-linux-clang-13-no_test_run: @@ -517,7 +534,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench + - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j8 all microbench - post-steps # Ensure ASAN+UBSAN with folly, and full testsuite with clang 13 @@ -528,7 +545,7 @@ jobs: - pre-steps - setup-folly - build-folly - - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check # TODO: LIB_MODE only to work around unresolved linker failures + - run: CC=clang-13 CXX=clang++-13 LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j8 check # TODO: LIB_MODE only to work around unresolved linker failures - post-steps # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing. @@ -537,7 +554,7 @@ jobs: resource_class: xlarge steps: - pre-steps - - run: DEBUG_LEVEL=0 make -j32 run_microbench + - run: DEBUG_LEVEL=0 make -j8 run_microbench - post-steps build-linux-mini-crashtest: @@ -555,7 +572,7 @@ jobs: - pre-steps - run: name: "run crashtest" - command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage + command: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' blackbox_crash_test_with_tiered_storage no_output_timeout: 100m - post-steps @@ -566,7 +583,7 @@ jobs: - pre-steps - run: name: "run crashtest" - command: ulimit -S -n `ulimit -H -n` && make V=1 -j32 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage + command: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=10800 --use_io_uring=0' whitebox_crash_test_with_tiered_storage no_output_timeout: 100m - post-steps @@ -641,26 +658,26 @@ jobs: command: make V=1 J=8 -j8 jtest - post-steps - build-linux-java-pmd: - machine: - image: ubuntu-2004:202111-02 - resource_class: large - environment: - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - steps: - - install-maven - - pre-steps - - run: - name: "Set Java Environment" - command: | - echo "JAVA_HOME=${JAVA_HOME}" - echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV - which java && java -version - which javac && javac -version - - run: - name: "PMD RocksDBJava" - command: make V=1 J=8 -j8 jpmd - - post-pmd-steps +# build-linux-java-pmd: +# machine: +# image: ubuntu-2004:202111-02 +# resource_class: large +# environment: +# JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 +# steps: +# - install-maven +# - pre-steps +# - run: +# name: "Set Java Environment" +# command: | +# echo "JAVA_HOME=${JAVA_HOME}" +# echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV +# which java && java -version +# which javac && javac -version +# - run: +# name: "PMD RocksDBJava" +# command: make V=1 J=8 -j8 jpmd +# - post-pmd-steps build-linux-java-static: executor: linux-java-docker @@ -788,7 +805,7 @@ jobs: TEST_TMPDIR: /tmp/rocksdb_test_tmp steps: - pre-steps - - run: make V=1 -j32 check + - run: make V=1 -j8 check - post-steps build-linux-arm-test-full: @@ -925,7 +942,7 @@ workflows: - build-macos-java - build-macos-java-static - build-macos-java-static-universal - - build-linux-java-pmd +# - build-linux-java-pmd jobs-macos: jobs: - build-macos diff --git a/java/rocksjni/flink_compactionfilterjni.cc b/java/rocksjni/flink_compactionfilterjni.cc index a45f12d7b..cd3e88027 100644 --- a/java/rocksjni/flink_compactionfilterjni.cc +++ b/java/rocksjni/flink_compactionfilterjni.cc @@ -20,8 +20,9 @@ class JniCallbackBase : public ROCKSDB_NAMESPACE::JniCallback { protected: inline void CheckAndRethrowException(JNIEnv* env) const { if (env->ExceptionCheck()) { + jthrowable obj = env->ExceptionOccurred(); env->ExceptionDescribe(); - env->Throw(env->ExceptionOccurred()); + env->Throw(obj); } } };