Merge branch 'main' into feature/cluster-info

.env.example

@@ -24,3 +24,7 @@ GT_KAFKA_ENDPOINTS = localhost:9092
 
 # Setting for fuzz tests
 GT_MYSQL_ADDR = localhost:4002
+
+# Setting for unstable fuzz tests
+GT_FUZZ_BINARY_PATH=/path/to/
+GT_FUZZ_INSTANCE_ROOT_DIR=/tmp/unstable_greptime

.github/actions/fuzz-test/action.yaml

@@ -3,11 +3,17 @@ description: 'Fuzz test given setup and service'
 inputs:
  target:
  description: "The fuzz target to test"
+ required: true
+ max-total-time:
+ description: "Max total time(secs)"
+ required: true
+ unstable:
+ default: 'false'
+ description: "Enable unstable feature"
 runs:
  using: composite
  steps:
  - name: Run Fuzz Test
  shell: bash
- run: cargo fuzz run ${{ inputs.target }} --fuzz-dir tests-fuzz -D -s none -- -max_total_time=120
- env:
- GT_MYSQL_ADDR: 127.0.0.1:4002
+ run: cargo fuzz run ${{ inputs.target }} --fuzz-dir tests-fuzz -D -s none ${{ inputs.unstable == 'true' && '--features=unstable' || '' }} -- -max_total_time=${{ inputs.max-total-time }}
+

.github/workflows/develop.yml

@@ -38,13 +38,20 @@ jobs:
  runs-on: ubuntu-20.04
  steps:
  - uses: actions/checkout@v4
- - uses: crate-ci/typos@v1.13.10
+ - uses: crate-ci/typos@master
  - name: Check the config docs
  run: |
  make config-docs && \
  git diff --name-only --exit-code ./config/config.md \
  || (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1)
 
+ license-header-check:
+ runs-on: ubuntu-20.04
+ name: Check License Header
+ steps:
+ - uses: actions/checkout@v4
+ - uses: korandoru/hawkeye@v5
+
  check:
  name: Check
  runs-on: ${{ matrix.os }}
@@ -130,7 +137,7 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table" ]
+ target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
  steps:
  - uses: actions/checkout@v4
  - uses: arduino/setup-protoc@v3
@@ -164,8 +171,62 @@ jobs:
  uses: ./.github/actions/fuzz-test
  env:
  CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+ GT_MYSQL_ADDR: 127.0.0.1:4002
+ with:
+ target: ${{ matrix.target }}
+ max-total-time: 120
+
+ unstable-fuzztest:
+ name: Unstable Fuzz Test
+ needs: build
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ target: [ "unstable_fuzz_create_table_standalone" ]
+ steps:
+ - uses: actions/checkout@v4
+ - uses: arduino/setup-protoc@v3
+ with:
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+ - uses: dtolnay/rust-toolchain@master
+ with:
+ toolchain: ${{ env.RUST_TOOLCHAIN }}
+ - name: Rust Cache
+ uses: Swatinem/rust-cache@v2
+ with:
+ # Shares across multiple jobs
+ shared-key: "fuzz-test-targets"
+ - name: Set Rust Fuzz
+ shell: bash
+ run: |
+ sudo apt update && sudo apt install -y libfuzzer-14-dev
+ cargo install cargo-fuzz
+ - name: Download pre-built binaries
+ uses: actions/download-artifact@v4
+ with:
+ name: bins
+ path: .
+ - name: Unzip binaries
+ run: tar -xvf ./bins.tar.gz
+ - name: Fuzz Test
+ uses: ./.github/actions/fuzz-test
+ env:
+ CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+ GT_MYSQL_ADDR: 127.0.0.1:4002
+ GT_FUZZ_BINARY_PATH: ./bins/greptime
+ GT_FUZZ_INSTANCE_ROOT_DIR: /tmp/unstable-greptime/
  with:
  target: ${{ matrix.target }}
+ max-total-time: 120
+ unstable: 'true'
+ - name: Upload unstable fuzz test logs
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: unstable-fuzz-logs
+ path: /tmp/unstable-greptime/
+ retention-days: 3
+
 
  sqlness:
  name: Sqlness Test

.github/workflows/docs.yml

@@ -34,7 +34,14 @@ jobs:
  runs-on: ubuntu-20.04
  steps:
  - uses: actions/checkout@v4
- - uses: crate-ci/typos@v1.13.10
+ - uses: crate-ci/typos@master
+
+ license-header-check:
+ runs-on: ubuntu-20.04
+ name: Check License Header
+ steps:
+ - uses: actions/checkout@v4
+ - uses: korandoru/hawkeye@v5
 
  check:
  name: Check

.github/workflows/nightly-ci.yml

@@ -1,5 +1,3 @@
-# Nightly CI: runs tests every night for our second tier plaforms (Windows)
-
 on:
  schedule:
  - cron: '0 23 * * 1-5'
@@ -15,13 +13,29 @@ env:
  RUST_TOOLCHAIN: nightly-2024-04-18
 
 jobs:
- sqlness:
- name: Sqlness Test
+ sqlness-test:
+ name: Run sqlness test
+ if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+ runs-on: ubuntu-22.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Run sqlness test
+ uses: ./.github/actions/sqlness-test
+ with:
+ data-root: sqlness-test
+ aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
+ aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+ aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
+
+ sqlness-windows:
+ name: Sqlness tests on Windows
  if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ windows-latest-8-cores ]
+ runs-on: windows-latest-8-cores
  timeout-minutes: 60
  steps:
  - uses: actions/checkout@v4
@@ -52,6 +66,7 @@ jobs:
  retention-days: 3
 
  test-on-windows:
+ name: Run tests on Windows
  if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
  runs-on: windows-latest-8-cores
  timeout-minutes: 60

CONTRIBUTING.md

@@ -50,7 +50,7 @@ GreptimeDB uses the [Apache 2.0 license](https://github.com/GreptimeTeam/greptim
 
 - To ensure that community is free and confident in its ability to use your contributions, please sign the Contributor License Agreement (CLA) which will be incorporated in the pull request process.
 - Make sure all files have proper license header (running `docker run --rm -v $(pwd):/github/workspace ghcr.io/korandoru/hawkeye-native:v3 format` from the project root).
-- Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/) and [style guide](http://github.com/greptimeTeam/docs/style-guide.md).
+- Make sure all your codes are formatted and follow the [coding style](https://pingcap.github.io/style-guide/rust/) and [style guide](docs/style-guide.md).
 - Make sure all unit tests are passed (using `cargo test --workspace` or [nextest](https://nexte.st/index.html) `cargo nextest run`).
 - Make sure all clippy warnings are fixed (you can check it locally by running `cargo clippy --workspace --all-targets -- -D warnings`).
 

Cargo.toml

@@ -225,6 +225,8 @@ sql = { path = "src/sql" }
 store-api = { path = "src/store-api" }
 substrait = { path = "src/common/substrait" }
 table = { path = "src/table" }
+# TODO some code depends on this
+tests-integration = { path = "tests-integration" }
 
 [workspace.dependencies.meter-macros]
 git = "https://github.com/GreptimeTeam/greptime-meter.git"

benchmarks/Cargo.toml

@@ -33,6 +33,8 @@ rand.workspace = true
 rskafka.workspace = true
 serde.workspace = true
 store-api.workspace = true
+# TODO depend `Database` client
+tests-integration.workspace = true
 tokio.workspace = true
 toml.workspace = true
 uuid.workspace = true

benchmarks/src/bin/nyc-taxi.rs

@@ -29,10 +29,11 @@ use client::api::v1::column::Values;
 use client::api::v1::{
  Column, ColumnDataType, ColumnDef, CreateTableExpr, InsertRequest, InsertRequests, SemanticType,
 };
-use client::{Client, Database, OutputData, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use client::{Client, OutputData, DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
 use futures_util::TryStreamExt;
 use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+use tests_integration::database::Database;
 use tokio::task::JoinSet;
 
 const CATALOG_NAME: &str = "greptime";