From 6e841db68ef193b2bdaf8d72ef987c81627c2b79 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 23 Jul 2024 13:21:18 +0200 Subject: [PATCH] fix(sourcebundle): Skip non-UTF8 files (#2109) --- Cargo.lock | 202 +++++++++++++++++++++++++++++++++------ src/utils/file_upload.rs | 19 +++- 2 files changed, 187 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d7efb45049..a16c3d975f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,6 +49,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -142,6 +154,15 @@ dependencies = [ "regex", ] +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arrayvec" version = "0.7.4" @@ -629,6 +650,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.68", +] + [[package]] name = "difflib" version = "0.4.0" @@ -687,6 +719,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.68", +] + [[package]] name = "dlv-list" version = "0.3.0" @@ -1003,19 +1046,19 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -dependencies = [ - "fallible-iterator 0.3.0", - "stable_deref_trait", -] +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "gimli" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "e2e1d97fbe9722ba9bbd0c97051c2956e726562b61f86a25a4360398a40edfc9" +dependencies = [ + "fallible-iterator 0.3.0", + "stable_deref_trait", +] [[package]] name = "git2" @@ -1051,13 +1094,13 @@ dependencies = [ [[package]] name = "goblin" -version = "0.7.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27c1b4369c2cd341b5de549380158b105a04c331be5db9110eef7b6d2742134" +checksum = "1b363a30c165f666402fe6a3024d3bec7ebc898f96a4a23bd1c99f8dbf3f4f47" dependencies = [ "log", "plain", - "scroll 0.11.0", + "scroll 0.12.0", ] [[package]] @@ -1066,7 +1109,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.8", ] [[package]] @@ -1074,6 +1117,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash 0.8.11", + "serde", +] [[package]] name = "heck" @@ -1219,6 +1266,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.5", + "serde", ] [[package]] @@ -1416,6 +1464,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -2258,8 +2312,14 @@ name = "scroll" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" + +[[package]] +name = "scroll" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6" dependencies = [ - "scroll_derive 0.11.1", + "scroll_derive 0.12.0", ] [[package]] @@ -2275,9 +2335,9 @@ dependencies = [ [[package]] name = "scroll_derive" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db149f81d46d2deba7cd3c50772474707729550221e69588478ebf9ada425ae" +checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" dependencies = [ "proc-macro2", "quote", @@ -2424,7 +2484,7 @@ dependencies = [ "walkdir", "which", "winapi 0.3.9", - "zip", + "zip 0.6.6", ] [[package]] @@ -2590,6 +2650,12 @@ dependencies = [ "outref", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "similar" version = "2.5.0" @@ -2710,9 +2776,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "symbolic" -version = "12.8.0" +version = "12.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ec4f53c56d7ee8809c2322925d362e193bcc7bbe7e777a3304b34ea7e85a36" +checksum = "85aabcf85c883278298596217d678c8d3ca256445d732eac59303ce04863c46f" dependencies = [ "symbolic-common", "symbolic-debuginfo", @@ -2735,9 +2801,9 @@ dependencies = [ [[package]] name = "symbolic-debuginfo" -version = "12.8.0" +version = "12.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb52777be67777947c5a159f1b6e8bfe4473d91fad7e5d4aff85ee4d3963cc04" +checksum = "abdc791ca87a69a5d09913d87f1e5ac95229be414ec0ff6c0fe2ddff6199f3b6" dependencies = [ "debugid", "dmsort", @@ -2745,7 +2811,7 @@ dependencies = [ "elsa", "fallible-iterator 0.3.0", "flate2", - "gimli 0.28.1", + "gimli 0.30.0", "goblin", "lazy_static", "nom", @@ -2754,7 +2820,7 @@ dependencies = [ "parking_lot", "pdb-addr2line", "regex", - "scroll 0.11.0", + "scroll 0.12.0", "serde", "serde_json", "smallvec", @@ -2762,14 +2828,15 @@ dependencies = [ "symbolic-ppdb", "thiserror", "wasmparser", - "zip", + "zip 2.1.1", + "zstd 0.13.1", ] [[package]] name = "symbolic-il2cpp" -version = "12.8.0" +version = "12.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8d453d3c853febd7dbfcb9a7ff18a94aa0d64f7a6476f8f453e88b01ee33d9" +checksum = "88cc2a0a415e3cb2dfb900f6f5b3abf5ccd356c236927b36b3d09046175acbde" dependencies = [ "indexmap", "serde_json", @@ -2795,9 +2862,9 @@ dependencies = [ [[package]] name = "symbolic-symcache" -version = "12.8.0" +version = "12.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5bddf351855de48a9b9deb1ef1a755e228ee2902fbeb8ef255e2522afd6dae" +checksum = "1cb772d8bea63f0cc1cbb0690bbd3d955a27746e380954d7e1a30e88d7aaecd5" dependencies = [ "indexmap", "symbolic-common", @@ -3208,12 +3275,16 @@ checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasmparser" -version = "0.118.2" +version = "0.209.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f1154f1ab868e2a01d9834a805faca7bf8b50d041b4ca714d005d0dab1c50c" +checksum = "07035cc9a9b41e62d3bb3a3815a66ab87c993c06fe1cf6b2a3f2a18499d937db" dependencies = [ + "ahash 0.8.11", + "bitflags 2.6.0", + "hashbrown 0.14.5", "indexmap", "semver 1.0.23", + "serde", ] [[package]] @@ -3505,6 +3576,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.68", +] + [[package]] name = "zip" version = "0.6.6" @@ -3522,7 +3613,38 @@ dependencies = [ "pbkdf2", "sha1", "time", - "zstd", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "zip" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd56a4d5921bc2f99947ac5b3abe5f510b1be7376fdc5e9fce4a23c6a93e87c" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", ] [[package]] @@ -3531,7 +3653,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +dependencies = [ + "zstd-safe 7.1.0", ] [[package]] @@ -3544,6 +3675,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.11+zstd.1.5.6" diff --git a/src/utils/file_upload.rs b/src/utils/file_upload.rs index c653a851bc..809b9a6cb5 100644 --- a/src/utils/file_upload.rs +++ b/src/utils/file_upload.rs @@ -9,13 +9,16 @@ use std::time::{Duration, Instant}; use anyhow::{anyhow, bail, Result}; use console::style; +use log::info; use parking_lot::RwLock; use rayon::prelude::*; use rayon::ThreadPoolBuilder; use sentry::types::DebugId; use sha1_smol::Digest; use symbolic::common::ByteView; -use symbolic::debuginfo::sourcebundle::{SourceBundleWriter, SourceFileInfo, SourceFileType}; +use symbolic::debuginfo::sourcebundle::{ + SourceBundleErrorKind, SourceBundleWriter, SourceFileInfo, SourceFileType, +}; use url::Url; use crate::api::NewRelease; @@ -519,7 +522,17 @@ fn build_artifact_bundle( } let bundle_path = url_to_bundle_path(&file.url)?; - bundle.add_file(bundle_path, file.contents.as_slice(), info)?; + if let Err(e) = bundle.add_file(bundle_path, file.contents.as_slice(), info) { + if e.kind() == SourceBundleErrorKind::ReadFailed { + info!( + "Skipping {} because it is not valid UTF-8.", + file.path.display() + ); + continue; + } else { + return Err(e.into()); + } + } } bundle.finish()?; @@ -673,7 +686,7 @@ mod tests { let hash = Sha1::from(buf); assert_eq!( hash.digest().to_string(), - "d38fb9915de70eec2aa2d0c380b344d89ef540f0" + "f0e25ae149b711c510148e022ebc883ad62c7c4c" ); } }