diff --git a/src/build.rs b/src/build.rs index c6199cfde..566d48ba1 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1,6 +1,8 @@ //! The build module contains the code for running the build process for a given [`Output`] use fs_err as fs; +use rattler_conda_types::{MatchSpec, ParseStrictness}; use std::path::PathBuf; +use std::vec; use miette::IntoDiagnostic; use rattler_index::index; @@ -8,22 +10,54 @@ use rattler_index::index; use crate::metadata::Output; use crate::package_test::TestConfiguration; use crate::recipe::parser::TestType; +use crate::render::solver::load_repodatas; use crate::{package_test, tool_configuration}; +/// Check if the build should be skipped because it already exists in any of the channels +pub async fn skip_existing( + output: &Output, + tool_configuration: &tool_configuration::Configuration, +) -> miette::Result { + // If we should skip existing builds, check if the build already exists + if tool_configuration.skip_existing { + let channels = output.reindex_channels().into_diagnostic()?; + let match_spec = + MatchSpec::from_str(output.name().as_normalized(), ParseStrictness::Strict) + .into_diagnostic()?; + let match_spec_vec = vec![match_spec.clone()]; + let (_, existing) = load_repodatas( + &channels, + output.target_platform(), + tool_configuration, + &match_spec_vec, + ) + .await + .unwrap(); + + return Ok(existing.iter().flatten().any(|package| { + package.package_record.version.to_string() == output.version() + && output.build_string() == Some(&package.package_record.build) + })); + } + Ok(false) +} + /// Run the build for the given output. This will fetch the sources, resolve the dependencies, /// and execute the build script. Returns the path to the resulting package. pub async fn run_build( output: Output, tool_configuration: &tool_configuration::Configuration, ) -> miette::Result<(Output, PathBuf)> { + if output.build_string().is_none() { + miette::bail!("Build string is not set for {:?}", output.name()); + } + output .build_configuration .directories .create_build_dir() .into_diagnostic()?; - if output.build_string().is_none() { - miette::bail!("Build string is not set for {:?}", output.name()); - } + let span = tracing::info_span!("Running build for", recipe = output.identifier().unwrap()); let _enter = span.enter(); output.record_build_start(); diff --git a/src/lib.rs b/src/lib.rs index fe46c5617..3691a66dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,7 @@ mod unix; pub mod upload; mod windows; +use build::skip_existing; use dunce::canonicalize; use fs_err as fs; use metadata::Output; @@ -123,6 +124,7 @@ pub fn get_tool_config( use_zstd: args.common.use_zstd, use_bz2: args.common.use_bz2, render_only: args.render_only, + skip_existing: args.skip_existing, } } @@ -305,6 +307,13 @@ pub async fn run_build_from_args( ) -> miette::Result<()> { let mut outputs: Vec = Vec::new(); for output in build_output { + if skip_existing(&output, &tool_config).await? { + tracing::info!( + "Skipping build for {:?}", + output.identifier().unwrap_or_else(|| "unknown".to_string()) + ); + continue; + } let output = match run_build(output, &tool_config).await { Ok((output, _archive)) => { output.record_build_end(); @@ -416,6 +425,7 @@ pub async fn rebuild_from_args( use_zstd: args.common.use_zstd, use_bz2: args.common.use_bz2, render_only: false, + skip_existing: false, }; output @@ -571,7 +581,7 @@ pub fn sort_build_outputs_topologically( .iter() .map(|idx| &outputs[idx.index()]) .for_each(|output| { - tracing::debug!("ordered output: {:?}", output.name().as_normalized()); + tracing::debug!("Ordered output: {:?}", output.name().as_normalized()); }); // Reorder outputs based on the sorted indices diff --git a/src/opt.rs b/src/opt.rs index a80b0c82b..f916e8701 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -253,6 +253,10 @@ pub struct BuildOpts { /// Launch the terminal user interface. #[arg(long, default_value = "false", hide = !cfg!(feature = "tui"))] pub tui: bool, + + /// Wether to skip packages that already exist in any channel + #[arg(long, default_value = "false")] + pub skip_existing: bool, } /// Test options. diff --git a/src/render/solver.rs b/src/render/solver.rs index 81e6ce628..f495a4723 100644 --- a/src/render/solver.rs +++ b/src/render/solver.rs @@ -71,13 +71,9 @@ pub async fn create_environment( channels: &[String], tool_configuration: &tool_configuration::Configuration, ) -> anyhow::Result> { - let channel_config = ChannelConfig::default(); // Parse the specs from the command line. We do this explicitly instead of allow clap to deal // with this because we need to parse the `channel_config` when parsing matchspecs. - // Find the default cache directory. Create it if it doesn't exist yet. - let cache_dir = rattler::default_cache_dir()?; - tracing::info!("\nResolving environment for:\n"); tracing::info!(" Platform: {}", target_platform); tracing::info!(" Channels: "); @@ -89,72 +85,12 @@ pub async fn create_environment( tracing::info!(" - {}", spec); } - std::fs::create_dir_all(&cache_dir) - .map_err(|e| anyhow::anyhow!("could not create cache directory: {}", e))?; - - // Determine the channels to use from the command line or select the default. Like matchspecs - // this also requires the use of the `channel_config` so we have to do this manually. - let channels = channels - .iter() - .map(|channel_str| Channel::from_str(channel_str, &channel_config)) - .collect::, _>>()?; - - // Each channel contains multiple subdirectories. Users can specify the subdirectories they want - // to use when specifying their channels. If the user didn't specify the default subdirectories - // we use defaults based on the current platform. - let platforms = [Platform::NoArch, *target_platform]; - let channel_urls = channels - .iter() - .flat_map(|channel| { - platforms - .iter() - .map(move |platform| (channel.clone(), *platform)) - }) - .collect::>(); - - // Determine the packages that are currently installed in the environment. let installed_packages = find_installed_packages(target_prefix, 100) .await .context("failed to determine currently installed packages")?; - // For each channel/subdirectory combination, download and cache the `repodata.json` that should - // be available from the corresponding Url. The code below also displays a nice CLI progress-bar - // to give users some more information about what is going on. - - let repodata_cache_path = cache_dir.join("repodata"); - let channel_and_platform_len = channel_urls.len(); - let repodata_download_client = tool_configuration.client.clone(); - let sparse_repo_datas = futures::stream::iter(channel_urls) - .map(move |(channel, platform)| { - let repodata_cache = repodata_cache_path.clone(); - let download_client = repodata_download_client.clone(); - async move { - fetch_repo_data_records_with_progress( - channel, - platform, - &repodata_cache, - download_client.clone(), - tool_configuration.fancy_log_handler.clone(), - platform != Platform::NoArch, - ) - .await - } - }) - .buffered(channel_and_platform_len) - .collect::>() - .await - // Collect into another iterator where we extract the first erroneous result - .into_iter() - .filter_map(Result::transpose) - .collect::, _>>()?; - - // Get the package names from the matchspecs so we can only load the package records that we need. - let package_names = specs.iter().filter_map(|spec| spec.name.clone()); - let repodatas = wrap_in_progress( - "parsing repodata", - &tool_configuration.fancy_log_handler, - move || SparseRepoData::load_records_recursive(&sparse_repo_datas, package_names, None), - )??; + let (cache_dir, repodatas) = + load_repodatas(channels, target_platform, tool_configuration, specs).await?; // Determine virtual packages of the system. These packages define the capabilities of the // system. Some packages depend on these virtual packages to indicate compatibility with the @@ -211,6 +147,71 @@ pub async fn create_environment( Ok(required_packages) } +/// Load repodata for given matchspecs and channels. +pub async fn load_repodatas( + channels: &[String], + target_platform: &Platform, + tool_configuration: &tool_configuration::Configuration, + specs: &[MatchSpec], +) -> Result<(PathBuf, Vec>), anyhow::Error> { + let channel_config = ChannelConfig::default(); + let cache_dir = rattler::default_cache_dir()?; + std::fs::create_dir_all(&cache_dir) + .map_err(|e| anyhow::anyhow!("could not create cache directory: {}", e))?; + + let channels = channels + .iter() + .map(|channel_str| Channel::from_str(channel_str, &channel_config)) + .collect::, _>>()?; + + let platforms = [Platform::NoArch, *target_platform]; + let channel_urls = channels + .iter() + .flat_map(|channel| { + platforms + .iter() + .map(move |platform| (channel.clone(), *platform)) + }) + .collect::>(); + + let repodata_cache_path = cache_dir.join("repodata"); + + let channel_and_platform_len = channel_urls.len(); + let repodata_download_client = tool_configuration.client.clone(); + let sparse_repo_datas = futures::stream::iter(channel_urls) + .map(move |(channel, platform)| { + let repodata_cache = repodata_cache_path.clone(); + let download_client = repodata_download_client.clone(); + async move { + fetch_repo_data_records_with_progress( + channel, + platform, + &repodata_cache, + download_client.clone(), + tool_configuration.fancy_log_handler.clone(), + platform != Platform::NoArch, + ) + .await + } + }) + .buffered(channel_and_platform_len) + .collect::>() + .await + // Collect into another iterator where we extract the first erroneous result + .into_iter() + .filter_map(Result::transpose) + .collect::, _>>()?; + + let package_names = specs.iter().filter_map(|spec| spec.name.clone()); + let repodatas = wrap_in_progress( + "parsing repodata", + &tool_configuration.fancy_log_handler, + move || SparseRepoData::load_records_recursive(&sparse_repo_datas, package_names, None), + )??; + + Ok((cache_dir, repodatas)) +} + pub async fn install_packages( required_packages: &Vec, target_platform: &Platform, diff --git a/src/tool_configuration.rs b/src/tool_configuration.rs index 63fc6678b..1756c8b87 100644 --- a/src/tool_configuration.rs +++ b/src/tool_configuration.rs @@ -33,6 +33,9 @@ pub struct Configuration { /// Whether to only render the build output pub render_only: bool, + + /// Wether to skip existing packages + pub skip_existing: bool, } /// Get the authentication storage from the given file @@ -77,6 +80,7 @@ impl Default for Configuration { use_zstd: true, use_bz2: true, render_only: false, + skip_existing: false, } } }