From b95dd014fa252d9cf43799a0ab6974cf766be9ec Mon Sep 17 00:00:00 2001 From: Xinye Tao Date: Wed, 14 Sep 2022 16:55:09 +0800 Subject: [PATCH] bump 0.3.0 (#256) * bump 0.3.0 Signed-off-by: tabokie * encode_len -> encoded_len Signed-off-by: tabokie * patch changelog Signed-off-by: tabokie * makefile force set toolchain if incompatible Signed-off-by: tabokie * simplify a bit Signed-off-by: tabokie * simplify make test command Signed-off-by: tabokie Signed-off-by: tabokie --- .github/workflows/rust.yml | 6 +++--- CHANGELOG.md | 3 ++- Cargo.toml | 2 +- Makefile | 39 +++++++++++++++++++++++++---------- README.md | 12 +++++------ ctl/Cargo.toml | 4 ++-- src/file_pipe_log/format.rs | 7 ++++--- src/file_pipe_log/log_file.rs | 8 +++---- src/file_pipe_log/mod.rs | 4 ++-- src/file_pipe_log/pipe.rs | 2 +- src/file_pipe_log/reader.rs | 2 +- stress/Cargo.toml | 2 +- 12 files changed, 55 insertions(+), 36 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 60e33efe..08b519dd 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -69,13 +69,13 @@ jobs: - name: Clippy run: make clippy env: - WITH_STABLE_TOOLCHAIN: 'true' + WITH_STABLE_TOOLCHAIN: 'force' - name: Run tests run: make test env: RUST_BACKTRACE: 1 EXTRA_CARGO_ARGS: '--verbose' - WITH_STABLE_TOOLCHAIN: 'true' + WITH_STABLE_TOOLCHAIN: 'force' coverage: runs-on: ubuntu-latest needs: nightly @@ -98,7 +98,7 @@ jobs: - name: Run tests run: | make test - env WITH_STABLE_TOOLCHAIN=true make test + env WITH_STABLE_TOOLCHAIN=auto make test env: RUSTFLAGS: '-Zinstrument-coverage' LLVM_PROFILE_FILE: '%p-%m.profraw' diff --git a/CHANGELOG.md b/CHANGELOG.md index 3294cfaf..5623af97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # Raft Engine Change Log -## [Unreleased] +## [0.3.0] - 2022-09-14 ### Bug Fixes * Unconditionally tolerate `fallocate` failures as a fix to its portability issue. Errors other than `EOPNOTSUPP` will still emit a warning. * Avoid leaving fractured write after failure by reseeking the file writer. Panic if the reseek fails as well. +* Fix a parallel recovery panic bug. * Fix panic when an empty batch is written to engine and then reused. ### New Features diff --git a/Cargo.toml b/Cargo.toml index af23c161..d7915ea5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "raft-engine" -version = "0.2.0" +version = "0.3.0" authors = ["The TiKV Project Developers"] edition = "2018" rust-version = "1.57" diff --git a/Makefile b/Makefile index aaeddcf0..968fd11d 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,28 @@ ## Additionaly arguments passed to cargo. EXTRA_CARGO_ARGS ?= -## Whether to disable nightly-only feature. [true/false] +## How to test stable toolchain. +## - auto: use current default toolchain, disable nightly features. +## - force: always use stable toolchain, disable nightly features. WITH_STABLE_TOOLCHAIN ?= +WITH_NIGHTLY_FEATURES = +ifeq (,$(filter $(WITH_STABLE_TOOLCHAIN),auto force)) +WITH_NIGHTLY_FEATURES = 1 +endif + +TOOLCHAIN_ARGS = +ifeq ($(shell (rustc --version | grep -q nightly); echo $$?), 1) +ifdef WITH_NIGHTLY_FEATURES +# Force use nightly toolchain if we are building with nightly features. +TOOLCHAIN_ARGS = +nightly +endif +else +ifeq ($(WITH_STABLE_TOOLCHAIN), force) +TOOLCHAIN_ARGS = +stable +endif +endif + .PHONY: format clippy test all: format clippy test @@ -14,21 +33,19 @@ format: cargo fmt --all ## Run clippy. -ifeq ($(WITH_STABLE_TOOLCHAIN), true) clippy: - cargo clippy --all --features all_stable --all-targets -- -D clippy::all +ifdef WITH_NIGHTLY_FEATURES + cargo ${TOOLCHAIN_ARGS} clippy --all --all-features --all-targets -- -D clippy::all else -clippy: - cargo clippy --all --all-features --all-targets -- -D clippy::all + cargo ${TOOLCHAIN_ARGS} clippy --all --features all_stable --all-targets -- -D clippy::all endif ## Run tests. -ifeq ($(WITH_STABLE_TOOLCHAIN), true) test: - cargo test --all --features all_stable_except_failpoints ${EXTRA_CARGO_ARGS} -- --nocapture - cargo test --test failpoints --features all_stable ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture +ifdef WITH_NIGHTLY_FEATURES + cargo ${TOOLCHAIN_ARGS} test --all --features all_except_failpoints ${EXTRA_CARGO_ARGS} -- --nocapture + cargo ${TOOLCHAIN_ARGS} test --test failpoints --all-features ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture else -test: - cargo test --all --features all_except_failpoints ${EXTRA_CARGO_ARGS} -- --nocapture - cargo test --test failpoints --all-features ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture + cargo ${TOOLCHAIN_ARGS} test --all --features all_stable_except_failpoints ${EXTRA_CARGO_ARGS} -- --nocapture + cargo ${TOOLCHAIN_ARGS} test --test failpoints --features all_stable ${EXTRA_CARGO_ARGS} -- --test-threads 1 --nocapture endif diff --git a/README.md b/README.md index 33c1e8ac..f70a1b1b 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Put this in your Cargo.toml: ```rust [dependencies] -raft-engine = "0.2.0" +raft-engine = "0.3.0" ``` Available Cargo features: @@ -74,13 +74,13 @@ Contributions are always welcome! Here are a few tips for making a PR: - All commits must be signed off (with `git commit -s`) to pass the [DCO check](https://probot.github.io/apps/dco/). - Tests are automatically run against the changes, some of them can be run locally: -``` -# rustup default nightly +```bash +# run tests with nightly features make -# rustup default stable -env WITH_STABLE_TOOLCHAIN=true make +# run tests on stable toolchain +make WITH_STABLE_TOOLCHAIN=force # filter a specific test case -env EXTRA_CARGO_ARGS= make test +make test EXTRA_CARGO_ARGS= ``` - For changes that might induce performance effects, please quote the targeted benchmark results in the PR description. In addition to micro-benchmarks, there is a standalone [stress test tool](https://github.com/tikv/raft-engine/tree/master/stress) which you can use to demonstrate the system performance. diff --git a/ctl/Cargo.toml b/ctl/Cargo.toml index e3be2d53..c8615a9f 100644 --- a/ctl/Cargo.toml +++ b/ctl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "raft-engine-ctl" -version = "0.2.0" +version = "0.3.0" authors = ["The TiKV Project Developers"] edition = "2018" rust-version = "1.57" @@ -11,4 +11,4 @@ license = "Apache-2.0" [dependencies] clap = { version = "3.1", features = ["derive", "cargo"] } env_logger = "0.9" -raft-engine = { path = "..", version = "0.2.0", features = ["scripting", "internals"] } +raft-engine = { path = "..", version = "0.3.0", features = ["scripting", "internals"] } diff --git a/src/file_pipe_log/format.rs b/src/file_pipe_log/format.rs index b1c7e4c3..787cd36c 100644 --- a/src/file_pipe_log/format.rs +++ b/src/file_pipe_log/format.rs @@ -92,10 +92,11 @@ pub(super) fn lock_file_path>(dir: P) -> PathBuf { path } -/// In-memory representation of `Format` in log files. +/// Log file format. It will be encoded to file header. #[derive(Copy, Clone, Debug, Eq, PartialEq, Default)] pub struct LogFileFormat { pub version: Version, + /// 0 stands for no alignment. pub alignment: u64, } @@ -116,12 +117,12 @@ impl LogFileFormat { } } - pub const fn max_encode_len() -> usize { + pub const fn max_encoded_len() -> usize { Self::header_len() + Self::payload_len(Version::V2) } /// Length of whole `LogFileFormat` written on storage. - pub fn encode_len(version: Version) -> usize { + pub fn encoded_len(version: Version) -> usize { Self::header_len() + Self::payload_len(version) } diff --git a/src/file_pipe_log/log_file.rs b/src/file_pipe_log/log_file.rs index f977f695..a2a6f53d 100644 --- a/src/file_pipe_log/log_file.rs +++ b/src/file_pipe_log/log_file.rs @@ -57,8 +57,8 @@ impl LogFileWriter { written: file_size, capacity: file_size, }; - // TODO: add tests for file_size in [header_len, max_encode_len]. - if file_size < LogFileFormat::encode_len(format.version) || force_reset { + // TODO: add tests for file_size in [header_len, max_encoded_len]. + if file_size < LogFileFormat::encoded_len(format.version) || force_reset { f.write_header(format)?; } else { f.writer.seek(SeekFrom::Start(file_size as u64))?; @@ -69,7 +69,7 @@ impl LogFileWriter { fn write_header(&mut self, format: LogFileFormat) -> Result<()> { self.writer.seek(SeekFrom::Start(0))?; self.written = 0; - let mut buf = Vec::with_capacity(LogFileFormat::encode_len(format.version)); + let mut buf = Vec::with_capacity(LogFileFormat::encoded_len(format.version)); format.encode(&mut buf)?; self.write(&buf, 0) } @@ -165,7 +165,7 @@ impl LogFileReader { /// to `0`, that is, the beginning of the file, to parse the /// related `[LogFileFormat]`. pub fn parse_format(&mut self) -> Result { - let mut container = vec![0; LogFileFormat::max_encode_len()]; + let mut container = vec![0; LogFileFormat::max_encoded_len()]; let size = self.read_to(0, &mut container)?; container.truncate(size); LogFileFormat::decode(&mut container.as_slice()) diff --git a/src/file_pipe_log/mod.rs b/src/file_pipe_log/mod.rs index b75a7168..2409faaa 100644 --- a/src/file_pipe_log/mod.rs +++ b/src/file_pipe_log/mod.rs @@ -315,8 +315,8 @@ pub mod debug { for from in formats { for to in formats { for shorter in [true, false] { - if LogFileFormat::encode_len(to.version) - < LogFileFormat::encode_len(from.version) + if LogFileFormat::encoded_len(to.version) + < LogFileFormat::encoded_len(from.version) { continue; } diff --git a/src/file_pipe_log/pipe.rs b/src/file_pipe_log/pipe.rs index 81f65c72..ad99904f 100644 --- a/src/file_pipe_log/pipe.rs +++ b/src/file_pipe_log/pipe.rs @@ -620,7 +620,7 @@ mod tests { let pipe_log = new_test_pipes(&cfg).unwrap(); assert_eq!(pipe_log.file_span(queue), (1, 1)); - let header_size = LogFileFormat::encode_len(cfg.format_version) as u64; + let header_size = LogFileFormat::encoded_len(cfg.format_version) as u64; // generate file 1, 2, 3 let content: Vec = vec![b'a'; 1024]; diff --git a/src/file_pipe_log/reader.rs b/src/file_pipe_log/reader.rs index c5ad7e62..a56528d7 100644 --- a/src/file_pipe_log/reader.rs +++ b/src/file_pipe_log/reader.rs @@ -50,7 +50,7 @@ impl LogItemBatchFileReader { format: LogFileFormat, reader: LogFileReader, ) -> Result<()> { - self.valid_offset = LogFileFormat::encode_len(format.version); + self.valid_offset = LogFileFormat::encoded_len(format.version); self.file_id = Some(file_id); self.format = Some(format); self.size = reader.file_size()?; diff --git a/stress/Cargo.toml b/stress/Cargo.toml index e973c609..e89c92a5 100644 --- a/stress/Cargo.toml +++ b/stress/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stress" -version = "0.2.0" +version = "0.3.0" authors = ["The TiKV Authors"] edition = "2018"