diff --git a/Cargo.lock b/Cargo.lock index 2e3dee9..abdb0ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -997,6 +997,17 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs-set-times" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033b337d725b97690d86893f9de22b67b80dcc4e9ad815f348254c38119db8fb" +dependencies = [ + "io-lifetimes", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fslock" version = "0.2.1" @@ -1622,6 +1633,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-lifetimes" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a611371471e98973dbcab4e0ec66c31a10bc356eeb4d54a0e05eac8158fe38c" + [[package]] name = "ipnet" version = "2.9.0" @@ -2332,6 +2349,7 @@ dependencies = [ "async-std", "clap", "clap-verbosity-flag", + "fs-set-times", "futures", "fxhash", "indicatif", @@ -2349,6 +2367,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "sha2", "tempfile", "tokio", "tokio-stream", @@ -2357,6 +2376,7 @@ dependencies = [ "tracing-log", "tracing-subscriber", "url", + "walkdir", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 24c19a0..eaf9eb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,10 @@ tracing-log = "0.2.0" url = "2.5.2" fxhash = "0.2.1" tempfile = "3.13.0" +walkdir = "2.5.0" +fs-set-times = "0.20.1" [dev-dependencies] async-std = "1.13.0" rstest = "0.23.0" +sha2 = "0.10.8" diff --git a/src/pack.rs b/src/pack.rs index 954a91e..03c7399 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -1,5 +1,6 @@ use std::{ collections::{HashMap, HashSet}, + fs::FileTimes, path::{Path, PathBuf}, sync::Arc, }; @@ -19,9 +20,11 @@ use rattler_lock::{CondaPackage, LockFile, Package}; use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage}; use reqwest_middleware::ClientWithMiddleware; use tokio_tar::Builder; +use walkdir::WalkDir; use crate::{ - get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH, + get_size, util::set_default_file_times, PixiPackMetadata, ProgressReporter, + CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH, }; use anyhow::anyhow; @@ -160,15 +163,33 @@ pub async fn pack(options: PackOptions) -> Result<()> { // Add pixi-pack.json containing metadata. tracing::info!("Creating pixi-pack.json file"); let metadata_path = output_folder.path().join(PIXI_PACK_METADATA_PATH); - let mut metadata_file = File::create(&metadata_path).await?; - let metadata = serde_json::to_string_pretty(&options.metadata)?; - metadata_file.write_all(metadata.as_bytes()).await?; + fs::write(metadata_path, metadata.as_bytes()).await?; // Create environment file. tracing::info!("Creating environment.yml file"); create_environment_file(output_folder.path(), conda_packages.iter().map(|(_, p)| p)).await?; + // Adjusting all timestamps of directories and files (excl. conda packages). + for entry in WalkDir::new(output_folder.path()) + .follow_links(true) + .into_iter() + .filter_map(|e| e.ok()) + { + match entry.path().extension().and_then(|e| e.to_str()) { + Some("bz2") | Some("conda") => continue, + _ => { + set_default_file_times(entry.path()).map_err(|e| { + anyhow!( + "could not set default file times for path {}: {}", + entry.path().display(), + e + ) + })?; + } + } + } + // Pack = archive the contents. tracing::info!("Creating archive at {}", options.output_file.display()); archive_directory(output_folder.path(), &options.output_file) @@ -247,6 +268,22 @@ async fn download_package( dest.write_all(&chunk).await?; } + // Adjust file metadata (timestamps). + let package_timestamp = package + .package_record() + .timestamp + .ok_or_else(|| anyhow!("could not read package timestamp"))?; + let file_times = FileTimes::new() + .set_modified(package_timestamp.into()) + .set_accessed(package_timestamp.into()); + + // Make sure to write all data and metadata to disk before modifying timestamp. + dest.sync_all().await?; + let dest_file = dest + .try_into_std() + .map_err(|e| anyhow!("could not read standard file: {:?}", e))?; + dest_file.set_times(file_times)?; + Ok(()) } @@ -306,7 +343,7 @@ async fn create_environment_file( environment.push_str(&format!(" - {}\n", match_spec_str)); } - fs::write(environment_path, environment) + fs::write(environment_path.as_path(), environment) .await .map_err(|e| anyhow!("Could not write environment file: {}", e))?; @@ -350,7 +387,7 @@ async fn create_repodata_files( let repodata_json = serde_json::to_string_pretty(&repodata) .map_err(|e| anyhow!("could not serialize repodata: {}", e))?; - fs::write(repodata_path, repodata_json) + fs::write(repodata_path.as_path(), repodata_json) .map_err(|e| anyhow!("could not write repodata: {}", e)) .await?; } diff --git a/src/util.rs b/src/util.rs index 9cc0c04..148b8b1 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,5 +1,6 @@ use std::{path::Path, time::Duration}; +use fs_set_times::{set_times, SystemTimeSpec}; use indicatif::{ProgressBar, ProgressStyle}; /// Progress reporter that wraps a progress bar with default styles. @@ -30,3 +31,14 @@ pub fn get_size>(path: P) -> std::io::Result { } Ok(size) } + +/// Set the modified, accessed, created time for a file. +pub fn set_default_file_times>(path: P) -> std::io::Result<()> { + tracing::debug!("Changing times for {:?}", path.as_ref()); + set_times( + path, + Some(SystemTimeSpec::Absolute(std::time::SystemTime::UNIX_EPOCH)), + Some(SystemTimeSpec::Absolute(std::time::SystemTime::UNIX_EPOCH)), + )?; + Ok(()) +} diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 260e180..e7b5951 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -1,5 +1,7 @@ #![allow(clippy::too_many_arguments)] +use sha2::{Digest, Sha256}; +use std::{fs, io}; use std::{path::PathBuf, process::Command}; use pixi_pack::{unarchive, PackOptions, PixiPackMetadata, UnpackOptions}; @@ -269,6 +271,37 @@ async fn test_pypi_ignore( assert_eq!(pack_result.is_err(), should_fail); } +fn sha256_digest_bytes(path: &PathBuf) -> String { + let mut hasher = Sha256::new(); + let mut file = fs::File::open(path).unwrap(); + let _bytes_written = io::copy(&mut file, &mut hasher).unwrap(); + let digest = hasher.finalize(); + format!("{:X}", digest) +} + +#[rstest] +#[tokio::test] +async fn test_reproducible_shasum(options: Options) { + let mut pack_options = options.pack_options; + let output_file1 = options.output_dir.path().join("environment1.tar"); + let output_file2 = options.output_dir.path().join("environment2.tar"); + + // First pack. + pack_options.output_file = output_file1.clone(); + let pack_result = pixi_pack::pack(pack_options.clone()).await; + assert!(pack_result.is_ok(), "{:?}", pack_result); + + // Second pack. + pack_options.output_file = output_file2.clone(); + let pack_result = pixi_pack::pack(pack_options).await; + assert!(pack_result.is_ok(), "{:?}", pack_result); + + assert_eq!( + sha256_digest_bytes(&output_file1), + sha256_digest_bytes(&output_file2) + ); +} + #[rstest] #[tokio::test] async fn test_non_authenticated(