diff --git a/Cargo.lock b/Cargo.lock index 90ac7be..ff75c76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2203,7 +2203,7 @@ dependencies = [ [[package]] name = "pixi-pack" -version = "0.1.2" +version = "0.1.3" dependencies = [ "anyhow", "async-std", diff --git a/Cargo.toml b/Cargo.toml index 595c7c0..9b53544 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pixi-pack" description = "A command line tool to pack and unpack conda environments for easy sharing" -version = "0.1.2" +version = "0.1.3" edition = "2021" [features] diff --git a/src/lib.rs b/src/lib.rs index e00142d..3d0ea11 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,10 +1,12 @@ mod pack; mod unpack; +mod util; pub use pack::{pack, PackOptions}; use rattler_conda_types::Platform; use serde::{Deserialize, Serialize}; pub use unpack::{unarchive, unpack, UnpackOptions}; +pub use util::{get_size, ProgressReporter}; pub const CHANNEL_DIRECTORY_NAME: &str = "channel"; pub const PIXI_PACK_METADATA_PATH: &str = "pixi-pack.json"; diff --git a/src/pack.rs b/src/pack.rs index bf899b1..4d26498 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -5,6 +5,7 @@ use std::{ }; use fxhash::FxHashMap; +use indicatif::HumanBytes; use rattler_index::{package_record_from_conda, package_record_from_tar_bz2}; use tokio::{ fs::{self, create_dir_all, File}, @@ -13,14 +14,15 @@ use tokio::{ use anyhow::Result; use futures::{stream, StreamExt, TryFutureExt, TryStreamExt}; -use indicatif::ProgressStyle; use rattler_conda_types::{package::ArchiveType, ChannelInfo, PackageRecord, Platform, RepoData}; use rattler_lock::{CondaPackage, LockFile, Package}; use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage}; use reqwest_middleware::ClientWithMiddleware; use tokio_tar::Builder; -use crate::{PixiPackMetadata, CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH}; +use crate::{ + get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH, +}; use anyhow::anyhow; /// Options for packing a pixi environment. @@ -89,31 +91,24 @@ pub async fn pack(options: PackOptions) -> Result<()> { // Download packages to temporary directory. tracing::info!( - "Downloading {} packages", + "Downloading {} packages...", conda_packages_from_lockfile.len() ); - let bar = indicatif::ProgressBar::new(conda_packages_from_lockfile.len() as u64); - bar.set_style( - ProgressStyle::with_template( - "[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}", - ) - .expect("could not set progress style") - .progress_chars("##-"), + eprintln!( + "⏳ Downloading {} packages...", + conda_packages_from_lockfile.len() ); - + let bar = ProgressReporter::new(conda_packages_from_lockfile.len() as u64); stream::iter(conda_packages_from_lockfile.iter()) .map(Ok) .try_for_each_concurrent(50, |package| async { download_package(&client, package, &channel_dir).await?; - - bar.inc(1); - + bar.pb.inc(1); Ok(()) }) .await .map_err(|e: anyhow::Error| anyhow!("could not download package: {}", e))?; - - bar.finish(); + bar.pb.finish_and_clear(); let mut conda_packages: Vec<(String, PackageRecord)> = Vec::new(); @@ -133,6 +128,8 @@ pub async fn pack(options: PackOptions) -> Result<()> { .map(|(p, t)| (PathBuf::from(format!("{}{}", p, t.extension())), t)) }) .collect(); + + tracing::info!("Injecting {} packages", injected_packages.len()); for (path, archive_type) in injected_packages { // step 1: Derive PackageRecord from index.json inside the package let package_record = match archive_type { @@ -157,9 +154,11 @@ pub async fn pack(options: PackOptions) -> Result<()> { } // Create `repodata.json` files. + tracing::info!("Creating repodata.json files"); create_repodata_files(conda_packages.iter(), &channel_dir).await?; // Add pixi-pack.json containing metadata. + tracing::info!("Creating pixi-pack.json file"); let metadata_path = output_folder.path().join(PIXI_PACK_METADATA_PATH); let mut metadata_file = File::create(&metadata_path).await?; @@ -167,13 +166,27 @@ pub async fn pack(options: PackOptions) -> Result<()> { metadata_file.write_all(metadata.as_bytes()).await?; // Create environment file. + tracing::info!("Creating environment.yml file"); create_environment_file(output_folder.path(), conda_packages.iter().map(|(_, p)| p)).await?; // Pack = archive the contents. + tracing::info!("Creating archive at {}", options.output_file.display()); archive_directory(output_folder.path(), &options.output_file) .await .map_err(|e| anyhow!("could not archive directory: {}", e))?; + let output_size = HumanBytes(get_size(&options.output_file)?).to_string(); + tracing::info!( + "Created pack at {} with size {}.", + options.output_file.display(), + output_size + ); + eprintln!( + "📦 Created pack at {} with size {}.", + options.output_file.display(), + output_size + ); + Ok(()) } diff --git a/src/unpack.rs b/src/unpack.rs index 32c16c6..c7dbd23 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -22,7 +22,8 @@ use tokio_tar::Archive; use url::Url; use crate::{ - PixiPackMetadata, CHANNEL_DIRECTORY_NAME, DEFAULT_PIXI_PACK_VERSION, PIXI_PACK_METADATA_PATH, + PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, DEFAULT_PIXI_PACK_VERSION, + PIXI_PACK_METADATA_PATH, }; /// Options for unpacking a pixi environment. @@ -41,6 +42,7 @@ pub async fn unpack(options: UnpackOptions) -> Result<()> { let channel_directory = unpack_dir.join(CHANNEL_DIRECTORY_NAME); + tracing::info!("Unarchiving pack to {}", unpack_dir.display()); unarchive(&options.pack_file, &unpack_dir) .await .map_err(|e| anyhow!("Could not unarchive: {}", e))?; @@ -49,10 +51,12 @@ pub async fn unpack(options: UnpackOptions) -> Result<()> { let target_prefix = options.output_directory.join("env"); + tracing::info!("Creating prefix at {}", target_prefix.display()); create_prefix(&channel_directory, &target_prefix) .await .map_err(|e| anyhow!("Could not create prefix: {}", e))?; + tracing::info!("Generating activation script"); create_activation_script( &options.output_directory, &target_prefix, @@ -61,6 +65,15 @@ pub async fn unpack(options: UnpackOptions) -> Result<()> { .await .map_err(|e| anyhow!("Could not create activation script: {}", e))?; + tracing::info!( + "Finished unpacking to {}.", + options.output_directory.display(), + ); + eprintln!( + "💫 Finished unpacking to {}.", + options.output_directory.display() + ); + Ok(()) } @@ -155,11 +168,16 @@ async fn create_prefix(channel_dir: &Path, target_prefix: &Path) -> Result<()> { .map_err(|e| anyhow!("could not create temporary directory: {}", e))? .into_path(); + eprintln!( + "⏳ Extracting and installing {} packages...", + packages.len() + ); + let reporter = ProgressReporter::new(packages.len() as u64); + // extract packages to cache + tracing::info!("Creating cache with {} packages", packages.len()); let package_cache = PackageCache::new(cache_dir); - let installer = Installer::default(); - let repodata_records: Vec = stream::iter(packages) .map(|(file_name, package_record)| { let cache_key = CacheKey::from(&package_record); @@ -189,6 +207,7 @@ async fn create_prefix(channel_dir: &Path, target_prefix: &Path) -> Result<()> { ) .await .map_err(|e| anyhow!("could not extract package: {}", e))?; + reporter.pb.inc(1); Ok::(repodata_record) } @@ -198,6 +217,8 @@ async fn create_prefix(channel_dir: &Path, target_prefix: &Path) -> Result<()> { .await?; // Invariant: all packages are in the cache + tracing::info!("Installing {} packages", repodata_records.len()); + let installer = Installer::default(); installer .with_package_cache(package_cache) .install(&target_prefix, repodata_records) diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..9cc0c04 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,32 @@ +use std::{path::Path, time::Duration}; + +use indicatif::{ProgressBar, ProgressStyle}; + +/// Progress reporter that wraps a progress bar with default styles. +pub struct ProgressReporter { + pub pb: ProgressBar, +} + +impl ProgressReporter { + pub fn new(length: u64) -> Self { + let pb = ProgressBar::new(length).with_style( + ProgressStyle::with_template("[{elapsed_precise}] {bar:40.cyan/blue} {msg}") + .expect("could not set progress style") + .progress_chars("##-"), + ); + pb.enable_steady_tick(Duration::from_millis(500)); + Self { pb } + } +} + +/// Get the size of a file or directory in bytes. +pub fn get_size>(path: P) -> std::io::Result { + let metadata = std::fs::metadata(&path)?; + let mut size = metadata.len(); + if metadata.is_dir() { + for entry in std::fs::read_dir(&path)? { + size += get_size(entry?.path())?; + } + } + Ok(size) +}