fix: Set timestamps correctly inside pack (#43)

* wip: fix shasum

* minor fix

* minor fix

* Add integration test

* Include output directory in iterator to patch mtimes

Cargo.toml

@@ -55,7 +55,10 @@ tracing-log = "0.2.0"
 url = "2.5.2"
 fxhash = "0.2.1"
 tempfile = "3.13.0"
+walkdir = "2.5.0"
+fs-set-times = "0.20.1"
 
 [dev-dependencies]
 async-std = "1.13.0"
 rstest = "0.23.0"
+sha2 = "0.10.8"

src/pack.rs

@@ -1,5 +1,6 @@
 use std::{
     collections::{HashMap, HashSet},
+    fs::FileTimes,
     path::{Path, PathBuf},
     sync::Arc,
 };
@@ -19,9 +20,11 @@ use rattler_lock::{CondaPackage, LockFile, Package};
 use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage};
 use reqwest_middleware::ClientWithMiddleware;
 use tokio_tar::Builder;
+use walkdir::WalkDir;
 
 use crate::{
-    get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH,
+    get_size, util::set_default_file_times, PixiPackMetadata, ProgressReporter,
+    CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH,
 };
 use anyhow::anyhow;
 
@@ -160,15 +163,33 @@ pub async fn pack(options: PackOptions) -> Result<()> {
     // Add pixi-pack.json containing metadata.
     tracing::info!("Creating pixi-pack.json file");
     let metadata_path = output_folder.path().join(PIXI_PACK_METADATA_PATH);
-    let mut metadata_file = File::create(&metadata_path).await?;
-
     let metadata = serde_json::to_string_pretty(&options.metadata)?;
-    metadata_file.write_all(metadata.as_bytes()).await?;
+    fs::write(metadata_path, metadata.as_bytes()).await?;
 
     // Create environment file.
     tracing::info!("Creating environment.yml file");
     create_environment_file(output_folder.path(), conda_packages.iter().map(|(_, p)| p)).await?;
 
+    // Adjusting all timestamps of directories and files (excl. conda packages).
+    for entry in WalkDir::new(output_folder.path())
+        .follow_links(true)
+        .into_iter()
+        .filter_map(|e| e.ok())
+    {
+        match entry.path().extension().and_then(|e| e.to_str()) {
+            Some("bz2") | Some("conda") => continue,
+            _ => {
+                set_default_file_times(entry.path()).map_err(|e| {
+                    anyhow!(
+                        "could not set default file times for path {}: {}",
+                        entry.path().display(),
+                        e
+                    )
+                })?;
+            }
+        }
+    }
+
     // Pack = archive the contents.
     tracing::info!("Creating archive at {}", options.output_file.display());
     archive_directory(output_folder.path(), &options.output_file)
@@ -247,6 +268,22 @@ async fn download_package(
         dest.write_all(&chunk).await?;
     }
 
+    // Adjust file metadata (timestamps).
+    let package_timestamp = package
+        .package_record()
+        .timestamp
+        .ok_or_else(|| anyhow!("could not read package timestamp"))?;
+    let file_times = FileTimes::new()
+        .set_modified(package_timestamp.into())
+        .set_accessed(package_timestamp.into());
+
+    // Make sure to write all data and metadata to disk before modifying timestamp.
+    dest.sync_all().await?;
+    let dest_file = dest
+        .try_into_std()
+        .map_err(|e| anyhow!("could not read standard file: {:?}", e))?;
+    dest_file.set_times(file_times)?;
+
     Ok(())
 }
 
@@ -306,7 +343,7 @@ async fn create_environment_file(
         environment.push_str(&format!("  - {}\n", match_spec_str));
     }
 
-    fs::write(environment_path, environment)
+    fs::write(environment_path.as_path(), environment)
         .await
         .map_err(|e| anyhow!("Could not write environment file: {}", e))?;
 
@@ -350,7 +387,7 @@ async fn create_repodata_files(
 
         let repodata_json = serde_json::to_string_pretty(&repodata)
             .map_err(|e| anyhow!("could not serialize repodata: {}", e))?;
-        fs::write(repodata_path, repodata_json)
+        fs::write(repodata_path.as_path(), repodata_json)
             .map_err(|e| anyhow!("could not write repodata: {}", e))
             .await?;
     }

src/util.rs

@@ -1,5 +1,6 @@
 use std::{path::Path, time::Duration};
 
+use fs_set_times::{set_times, SystemTimeSpec};
 use indicatif::{ProgressBar, ProgressStyle};
 
 /// Progress reporter that wraps a progress bar with default styles.
@@ -30,3 +31,14 @@ pub fn get_size<P: AsRef<Path>>(path: P) -> std::io::Result<u64> {
     }
     Ok(size)
 }
+
+/// Set the modified, accessed, created time for a file.
+pub fn set_default_file_times<P: AsRef<Path>>(path: P) -> std::io::Result<()> {
+    tracing::debug!("Changing times for {:?}", path.as_ref());
+    set_times(
+        path,
+        Some(SystemTimeSpec::Absolute(std::time::SystemTime::UNIX_EPOCH)),
+        Some(SystemTimeSpec::Absolute(std::time::SystemTime::UNIX_EPOCH)),
+    )?;
+    Ok(())
+}

tests/integration_test.rs

@@ -1,5 +1,7 @@
 #![allow(clippy::too_many_arguments)]
 
+use sha2::{Digest, Sha256};
+use std::{fs, io};
 use std::{path::PathBuf, process::Command};
 
 use pixi_pack::{unarchive, PackOptions, PixiPackMetadata, UnpackOptions};
@@ -269,6 +271,37 @@ async fn test_pypi_ignore(
     assert_eq!(pack_result.is_err(), should_fail);
 }
 
+fn sha256_digest_bytes(path: &PathBuf) -> String {
+    let mut hasher = Sha256::new();
+    let mut file = fs::File::open(path).unwrap();
+    let _bytes_written = io::copy(&mut file, &mut hasher).unwrap();
+    let digest = hasher.finalize();
+    format!("{:X}", digest)
+}
+
+#[rstest]
+#[tokio::test]
+async fn test_reproducible_shasum(options: Options) {
+    let mut pack_options = options.pack_options;
+    let output_file1 = options.output_dir.path().join("environment1.tar");
+    let output_file2 = options.output_dir.path().join("environment2.tar");
+
+    // First pack.
+    pack_options.output_file = output_file1.clone();
+    let pack_result = pixi_pack::pack(pack_options.clone()).await;
+    assert!(pack_result.is_ok(), "{:?}", pack_result);
+
+    // Second pack.
+    pack_options.output_file = output_file2.clone();
+    let pack_result = pixi_pack::pack(pack_options).await;
+    assert!(pack_result.is_ok(), "{:?}", pack_result);
+
+    assert_eq!(
+        sha256_digest_bytes(&output_file1),
+        sha256_digest_bytes(&output_file2)
+    );
+}
+
 #[rstest]
 #[tokio::test]
 async fn test_non_authenticated(