From f3c05f9e9a72a6fd41806ebfcbfa618e1c3369a2 Mon Sep 17 00:00:00 2001 From: Bohdan Siryk Date: Thu, 5 Sep 2024 13:28:51 +0300 Subject: [PATCH 1/3] build_and_test workflow updated to allow clippy process all crate targets before it fails --- .github/workflows/build_and_test.yaml | 30 +++++++++++++++--- brro-compressor/tests/e2e.rs | 0 .../tests/wbros/go_gc_duration_count.wbro | Bin 0 -> 24060 bytes 3 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 brro-compressor/tests/e2e.rs create mode 100644 brro-compressor/tests/wbros/go_gc_duration_count.wbro diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 6355d40..5971c9e 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -35,9 +35,9 @@ jobs: - uses: Swatinem/rust-cache@v2 with: # rust-cache already handles all the sane defaults for caching rust builds. - # However because we are running seperate debug/release builds in parallel, - # we also need to add the runner and cargo_flags to the key so that a seperate cache is used. - # Otherwise only the last build to finish would get saved to the cache. + # However, because we are running separate debug/release builds in parallel, + # we also need to add the runner and cargo_flags to the key so that a separate cache is used. + # Otherwise, only the last build to finish would get saved to the cache. key: ${{ matrix.runner }} - ${{ matrix.cargo_flags }} - name: Install external deps for the prom-remote-api crate run: sudo apt-get install protobuf-compiler @@ -48,8 +48,28 @@ jobs: - name: Ensure that all crates compile and have no warnings under every possible combination of features # some things to explicitly point out: # * clippy also reports rustc warnings and errors - # * clippy --all-targets causes clippy to run against tests and examples which it doesnt do by default. - run: cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }} -- -D warnings + # * clippy --all-targets causes clippy to run against tests and examples which it doesn't do by default. + run: | + # Initialize a variable to track if any package fails + failure_occurred=false + + # Loop over each package in the workspace + for package in $(cargo metadata --format-version=1 --no-deps | jq -r '.packages[].name'); do + echo "Running clippy for package: $package" + cargo hack clippy --package "$package" --feature-powerset --all-targets --locked ${{ matrix.cargo_flags }} -- -D warnings + exit_code=$? + + # If clippy fails, mark failure_occurred as true + if [ $exit_code -ne 0 ]; then + failure_occurred=true + fi + done + + # If any package failed, exit with a non-zero status + if [ "$failure_occurred" = true ]; then + echo "Clippy failed on one or more packages." + exit 1 + fi - name: Ensure that tests pass run: cargo test ${{ matrix.cargo_flags }} --all-features --all-targets -- --nocapture - name: Ensure that tests did not create or modify any files that arent .gitignore'd diff --git a/brro-compressor/tests/e2e.rs b/brro-compressor/tests/e2e.rs new file mode 100644 index 0000000..e69de29 diff --git a/brro-compressor/tests/wbros/go_gc_duration_count.wbro b/brro-compressor/tests/wbros/go_gc_duration_count.wbro new file mode 100644 index 0000000000000000000000000000000000000000..71e88ae2f4ddfc0aa93b18c0eed8b9538c301428 GIT binary patch literal 24060 zcmbu;NsePh6hP4@5E2p-4;V0D5JCu*5JIN!At?H`OwlA=LD2VLdWLTaUP6c!cwD1c zf|u~c3cLeTs=A(}l*;riPmX?OT)!U~8Asv%c5yb^G_U-oyJ_@1?iS=@0cj zx|6)0-|u~Z=RkjovI_p#P)~D&M&(K>d zy)@~p@htPDNoS4o%$FveHJ)R>H0iAIJoBYVXN?z_FHJgYyvTfM(plpr=1Y^#8ZR?n znsnB9h56E?v&O5;mnNMxUSqyA>8$ZO^QB2=jW?JtO*(75$$V+jS>poprAcRv1@onq z&YJYrL2r$>*uS*WS(Dy6=&f;){YxvIHR-K`-WqSSe`%#VtI2xnptr_5>{nXptVwSj z^wxNn{YxvIHR-K`-Wu<*e`%$&CcSmgAMDY!eUCF5@AG$(R=UJ-%=m!sKf%h+PcvC( z9rV`tkbOuioi*uCbFj`DAF)qqrL!izb|a{xtVzFj@O-H8Df^ICI&0EjuY>QgM&s`wEp*mOZ*BC}q_=i@>!7zz^w#){^OqJn zYo)g~dTY{KJH2(#TPJ#Je9rkx3!SynTN}MK>8+jKI_Rwvy*0k5arGlDbk<65ZS>Zp zw|08#ptnx+*7%b1mlir}rMEVEYtmagy>-xACwgmq#raDMowd?i8@)B@t)1RF=&cjI zHNNKjrG?H~>8*|4n)KFAZyog3_=fY77CLLCw>Elf(px*db(t&QHA^wv&q9rV_T-WuO?{?bBct@PGLZ%uk@r?(FJ z>viILtnmZ?2hu`kt@PGLZ%uk@r?(Dz>qKvjA31+%p|e(cYooU&y|vR@2fcNox5iJL zzqHU8*p_I?-F>XU<<*=&Y6A+UO58>GpP>t%Ke=(Ocse&QV(Etd-u{ z=&ea_?ex|`Z=L9^vE=-vGdgRbx6bLUmEO9bw>Em~lHQv1))l?A(_7c{)Z z>z3XczjFVjGdgRbx6bLUmEO9bw>Em~lHQv1))l?A(_7c{)Z>z3Xczj6Pi zGdgRbx6bLUmEO9bw>Em~lHQv1))l?A(_7c{)Z>z3XczjOc3=!|uH3(wX$ zy|vO?7xdOfZ(Y(`lis?bw|08#n%+9-ts8plL~q^FTjLMzuXILdE%eqoy|vO?7xdOf zZ(Y(`lis?bw|08#n%+9-ts8plL~q^FTjNjezjQ`tE%eqoy|vO?7xdOfZ(Y(`lis?b zw|08#n%+9-ts8plL~q^FTjMY8zjQ`tE%eqoy|vO?7xdOfZ(Y(`lis?bw|08#n%+9- ztsDA7-O^d(Z|<#hMrSSb);Ybk(pwkw)<$n#(p!_>x}vvsdh43rI_Rw%dh0}Q-O^j* zAMU?&MrSSb);Ybk(pwkw)<$n#(p!_>x}vvsdh43rI_M8@{_k!VH%FZNZ@0UAW5jm1 P+uaZmmv4!<>5BgY-ia#M literal 0 HcmV?d00001 From 09acfa46a73694875c4d56033c5dce901b8a728c Mon Sep 17 00:00:00 2001 From: Bohdan Siryk Date: Thu, 5 Sep 2024 13:37:44 +0300 Subject: [PATCH 2/3] brro-compressor e2e tests --- brro-compressor/tests/e2e.rs | 149 ++++++++++++++++++ .../tests/wbros/go_gc_duration_count.wbro | Bin 24060 -> 0 bytes .../tests/wbros/go_gc_heap_goal_bytes.wbro | Bin 0 -> 23668 bytes 3 files changed, 149 insertions(+) delete mode 100644 brro-compressor/tests/wbros/go_gc_duration_count.wbro create mode 100644 brro-compressor/tests/wbros/go_gc_heap_goal_bytes.wbro diff --git a/brro-compressor/tests/e2e.rs b/brro-compressor/tests/e2e.rs index e69de29..714df5f 100644 --- a/brro-compressor/tests/e2e.rs +++ b/brro-compressor/tests/e2e.rs @@ -0,0 +1,149 @@ +use brro_compressor::utils::error::calculate_error; +use std::fs; +use std::path::{Path, PathBuf}; +use wavbrro::wavbrro::WavBrro; + +const TEST_FILE_NAME: &str = "go_gc_heap_goal_bytes.wbro"; +const TEST_COMPRESSED_FILE_NAME: &str = "go_gc_heap_goal_bytes.bro"; +const TEST_WBRO_PATH: &str = "tests/wbros/go_gc_heap_goal_bytes.wbro"; + +#[test] +fn test_compressor_idw_lossless() { + test_lossless_compression("idw") +} + +#[test] +fn test_compressor_idw_lossy() { + test_lossy_compression("fft") +} + +#[test] +fn test_compressor_polynomial_lossless() { + test_lossless_compression("polynomial") +} + +#[test] +fn test_compressor_polynomial_lossy() { + test_lossy_compression("fft") +} + +#[test] +fn test_compressor_noop() { + test_lossless_compression("noop") +} + +#[test] +fn test_compressor_fft_lossy() { + test_lossy_compression("fft") +} + +#[test] +fn test_compressor_auto_lossless() { + test_lossless_compression("auto") +} + +#[test] +fn test_compressor_auto_lossy() { + test_lossy_compression("auto") +} + +fn test_lossless_compression(compressor: &str) { + test_compression_decompression_flow(compressor, 0, compare_samples_lossless) +} + +fn test_lossy_compression(compressor: &str) { + test_compression_decompression_flow(compressor, 5, compare_samples_with_allowed_error) +} + +#[test] +fn test_compressor_constant() { + // tests/wbros/uptime.wbro constant data which can be compressed by constant compressor + let test_dir = tempfile::tempdir().unwrap().into_path(); + fs::copy("tests/wbros/uptime.wbro", test_dir.join("uptime.wbro")).unwrap(); + + run_compressor(&[ + "--compressor", + "noop", + test_dir.join("uptime.wbro").to_str().unwrap(), + ]); + + run_compressor(&["-u", test_dir.join("uptime.bro").to_str().unwrap()]); + + compare_samples_lossless( + &PathBuf::from("tests/wbros/uptime.wbro"), + &test_dir.join("uptime.wbro"), + ) +} + +/// Runs compression and decompression test for a specified compressor. +/// max_error is an error level, compression speed is set as the lowest (0). +/// +/// It compresses tests/wbros/go_gc_duration_count.wbro file, decompresses +/// got .bro file, and compares the original .wbro and the decompressed one. +fn test_compression_decompression_flow( + compressor: &str, + allowed_error: u8, + compare_fn: fn(original: &Path, processed: &Path), +) { + let test_dir = prepare_test_dir(); + + // Running data compression + run_compressor(&[ + "--compressor", + compressor, + "--error", + allowed_error.to_string().as_str(), + test_dir.join(TEST_FILE_NAME).to_str().unwrap(), + ]); + + // Running data decompression + run_compressor(&[ + "-u", + test_dir.join(TEST_COMPRESSED_FILE_NAME).to_str().unwrap(), + ]); + + compare_fn( + &PathBuf::from(TEST_WBRO_PATH), + &test_dir.join(TEST_FILE_NAME), + ) +} + +/// Prepares test directory and copies test wbro file there. +fn prepare_test_dir() -> PathBuf { + let test_dir = tempfile::tempdir().unwrap().into_path(); + fs::copy(TEST_WBRO_PATH, test_dir.join(TEST_FILE_NAME)).unwrap(); + test_dir +} + +/// Runs compressor binary with provided arguments. +fn run_compressor(args: &[&str]) { + let compressor_bin = env!("CARGO_BIN_EXE_brro-compressor"); + let exit_status = std::process::Command::new(compressor_bin) + .args(args) + .status() + .unwrap(); + + assert!(exit_status.success()); +} + +fn compare_samples_lossless(original: &Path, uncompressed: &Path) { + let original_samples = WavBrro::from_file(original).unwrap(); + let uncompressed_samples = WavBrro::from_file(uncompressed).unwrap(); + assert_eq!(original_samples, uncompressed_samples); +} + +fn compare_samples_with_allowed_error(original: &Path, uncompressed: &Path) { + const MAX_ALLOWED_ERROR: f64 = 0.05; + + let original_samples = WavBrro::from_file(original).unwrap(); + let uncompressed_samples = WavBrro::from_file(uncompressed).unwrap(); + let err = calculate_error(&original_samples, &uncompressed_samples); + + assert!( + err <= MAX_ALLOWED_ERROR, + "Error: {}\nOriginal : {:?}\nUncompressed: {:?}", + err, + original_samples, + uncompressed_samples + ); +} diff --git a/brro-compressor/tests/wbros/go_gc_duration_count.wbro b/brro-compressor/tests/wbros/go_gc_duration_count.wbro deleted file mode 100644 index 71e88ae2f4ddfc0aa93b18c0eed8b9538c301428..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24060 zcmbu;NsePh6hP4@5E2p-4;V0D5JCu*5JIN!At?H`OwlA=LD2VLdWLTaUP6c!cwD1c zf|u~c3cLeTs=A(}l*;riPmX?OT)!U~8Asv%c5yb^G_U-oyJ_@1?iS=@0cj zx|6)0-|u~Z=RkjovI_p#P)~D&M&(K>d zy)@~p@htPDNoS4o%$FveHJ)R>H0iAIJoBYVXN?z_FHJgYyvTfM(plpr=1Y^#8ZR?n znsnB9h56E?v&O5;mnNMxUSqyA>8$ZO^QB2=jW?JtO*(75$$V+jS>poprAcRv1@onq z&YJYrL2r$>*uS*WS(Dy6=&f;){YxvIHR-K`-WqSSe`%#VtI2xnptr_5>{nXptVwSj z^wxNn{YxvIHR-K`-Wu<*e`%$&CcSmgAMDY!eUCF5@AG$(R=UJ-%=m!sKf%h+PcvC( z9rV`tkbOuioi*uCbFj`DAF)qqrL!izb|a{xtVzFj@O-H8Df^ICI&0EjuY>QgM&s`wEp*mOZ*BC}q_=i@>!7zz^w#){^OqJn zYo)g~dTY{KJH2(#TPJ#Je9rkx3!SynTN}MK>8+jKI_Rwvy*0k5arGlDbk<65ZS>Zp zw|08#ptnx+*7%b1mlir}rMEVEYtmagy>-xACwgmq#raDMowd?i8@)B@t)1RF=&cjI zHNNKjrG?H~>8*|4n)KFAZyog3_=fY77CLLCw>Elf(px*db(t&QHA^wv&q9rV_T-WuO?{?bBct@PGLZ%uk@r?(FJ z>viILtnmZ?2hu`kt@PGLZ%uk@r?(Dz>qKvjA31+%p|e(cYooU&y|vR@2fcNox5iJL zzqHU8*p_I?-F>XU<<*=&Y6A+UO58>GpP>t%Ke=(Ocse&QV(Etd-u{ z=&ea_?ex|`Z=L9^vE=-vGdgRbx6bLUmEO9bw>Em~lHQv1))l?A(_7c{)Z z>z3XczjFVjGdgRbx6bLUmEO9bw>Em~lHQv1))l?A(_7c{)Z>z3Xczj6Pi zGdgRbx6bLUmEO9bw>Em~lHQv1))l?A(_7c{)Z>z3XczjOc3=!|uH3(wX$ zy|vO?7xdOfZ(Y(`lis?bw|08#n%+9-ts8plL~q^FTjLMzuXILdE%eqoy|vO?7xdOf zZ(Y(`lis?bw|08#n%+9-ts8plL~q^FTjNjezjQ`tE%eqoy|vO?7xdOfZ(Y(`lis?b zw|08#n%+9-ts8plL~q^FTjMY8zjQ`tE%eqoy|vO?7xdOfZ(Y(`lis?bw|08#n%+9- ztsDA7-O^d(Z|<#hMrSSb);Ybk(pwkw)<$n#(p!_>x}vvsdh43rI_Rw%dh0}Q-O^j* zAMU?&MrSSb);Ybk(pwkw)<$n#(p!_>x}vvsdh43rI_M8@{_k!VH%FZNZ@0UAW5jm1 P+uaZmmv4!<>5BgY-ia#M diff --git a/brro-compressor/tests/wbros/go_gc_heap_goal_bytes.wbro b/brro-compressor/tests/wbros/go_gc_heap_goal_bytes.wbro new file mode 100644 index 0000000000000000000000000000000000000000..d6741a97e8168118e3d08aab8c95cf62001f0434 GIT binary patch literal 23668 zcmbu{e{c?W9tZGkjfz%zZhEJNV%r{dSKQJILa=CD6U}7>P1>8biyL)RY>-IF>0Uz# zp&P$@Q>>h7a#7tLis{h3sGL@RJTpq-nCn(XiyjtrF&F2#Jnv6(bJLj)dGpU}V)A}I z-(S1Wcc0xSStBzuw8no~jVg-b&vJ4{G^);8o%jC+*7^DIpIOx>w=BTxecvO;3h@6; zx?OwCi~syT9~*_bZj*MwPW*hUn*T|*@p`#$pg*ql4eIG9K1F+)HGY}-hxqR)NjvcS z%|>^AhyTA-)!xU7(4Q9S|7sO-wAm9SPipKOxG3IkR)5Oyh4Fk#HSWkjyMM@F#^G1u zuiLCv^%j0^sc!OEw2#$RV`wLB>6@r;x77RFz8depP3jc5sLL8g_AZInznPJfI)3kF zD%Ldgr=_Z=QtasLvl`}pj6bUtK8pIRUc)}DKi04;5%pQknH!L2HU4o`#*fs}ktpEVraYboA;NTYKH zpgwIwJIRxp%hzK4vMM8+=AYDmFoyYKbsxABZk&uCsoSlc)Mq{TCF;AR(adUT zKdIesI`XX1qCrxhRBt~OdD`>yF#h^2>JM0ldaPE5i^#Kj>AR3;Ri^Yre_8E)4LIId zwXKP0KdZW;6nWZS&q7dL_}yryKIc7L7dLBc ze**P?NIiBM^K&!Ra2tM)H5j=T?PK*5CnC>kJS68`R=wN%a@^mPj@%do&>fXgT zv-&xw@&3vhbeV_xtdWu_$g_q^cOcK|tx3f?VT~M0m-drt|9%g7R`aF($g_HiM7pSQ4vXV2hxVzn2fqy4PLj=y0Yu&U`FpgybeNdoG# zs{UlD&pM$4>Z_#DdI$AcjlFd;{-oNMD(bVU+A!%qYt;=If7WLa!XbDrNpjwMBCM#_C@`i}tf>#eYPe)hhOoXLUX^F@CK2{rRX*yJD2=+eyvB zLgZP!{_=S#tFdP|`p;_TPDcA#BdOg{pEYRHaanx-#OnP1DC)Cn!=FO`S^d1NSih`6 z$sDwwHCid3_p(~mOVEE-`@0orKdU*s0(n;bwQ97V)ySET`mAbdAJk{{gQoo}saGrK zZ`zu^Xun6Q^gf3?tNLMg89&m8Q8L?+?~{3 z^AkD$kcOR3VfrF|0xBt5Tm-pWOv{R(oy= z)&Z+hb6D0fsZ;qP@~q~)avz&D`1ujkXZ0>Ti*?8v*2!}ttVY&xw4YT;|61mcwcU8E zKZVrXcLDWT!wct-XEhR^m+>PF3R2|wCH30oAZi!F zn*S_9o;A2J1LtqrAsbLXAXVqiDT!Z4(k55Qbu_7wAfP;#-m@H7=FmdX_ah>3!5^RUU4G@nhBNm!dwa-MSX_S-syzQJ+<* z>n!~z^a%*8-^lS#YPOw*_Ooh*ImolBRsCiB zNzIqHp#QAa%d=(tNsWJxE*&UH*wKE1{{8`NxH=#bOp7vLaA8RmI?q{)@ zy=u^ZR=rm4=dng7{|oh5!)Mnc&#FGz4|!IlQ2NiRO-e?6*6^9JSU;@3yB+me?c%9e z|E$(4Rmig%CDYMWEW2UVi~tWLgM|FXJ$|Bm{sYPIZNS(QGQP@mP?BiFyI<|CPyf7VD$t{+*wyJw<4 ztJC8Xj32AHbd2<$)EQla_0JmVE$?5f(W)BspEbBB&!ez99am%gSiQqhw4c>j{3hzN zYBLV$n4?U98TH=8uNQ0_ahnoZA7vqmZ=qW`S=lBT~qMXHuw zM19&3mn2W>Z#jthW7S@e@5QqQrScput1|v^^q_MK@_~NpRAE}Zq&jGU<$K<{R?a*Vg{z=1+ z1s#jP6ucyKHhyR-3Otm1nVrFEnK1rSl} AR{#J2 literal 0 HcmV?d00001 From 5d553e3b47abf708bbb5c1a8defeb3477a30f6d0 Mon Sep 17 00:00:00 2001 From: Bohdan Siryk Date: Tue, 24 Sep 2024 14:43:00 +0300 Subject: [PATCH 3/3] CI clippy lint improvement --- .github/workflows/build_and_test.yaml | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index ae01834..5314356 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -50,26 +50,10 @@ jobs: # * clippy also reports rustc warnings and errors # * clippy --all-targets causes clippy to run against tests and examples which it doesn't do by default. run: | - # Initialize a variable to track if any package fails - failure_occurred=false - - # Loop over each package in the workspace - for package in $(cargo metadata --format-version=1 --no-deps | jq -r '.packages[].name'); do - echo "Running clippy for package: $package" - cargo hack clippy --package "$package" --feature-powerset --all-targets --locked ${{ matrix.cargo_flags }} -- -D warnings - exit_code=$? - - # If clippy fails, mark failure_occurred as true - if [ $exit_code -ne 0 ]; then - failure_occurred=true - fi - done - - # If any package failed, exit with a non-zero status - if [ "$failure_occurred" = true ]; then - echo "Clippy failed on one or more packages." - exit 1 - fi + # Display all clippy lint failures as warnings + cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }} + # Fail CI on the first crate to fail clippy lints + cargo hack --feature-powerset clippy --all-targets --locked ${{ matrix.cargo_flags }} -- -D warnings - name: Ensure that tests pass run: | cargo test --doc ${{ matrix.cargo_flags }} --all-features -- --show-output --nocapture