Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: Remove zstd compression #20

Merged
merged 1 commit into from
Jun 6, 2024
Merged

chore: Remove zstd compression #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,18 @@ $RECYCLE.BIN/
*.egg-info

# pixi-pack
environment.tar.zstd
environment.tar
environment
activate.*
cache/
channel/
env/
environment.yml
environment
environment.tar
environment.tar.zst
environment.tar.zstd
pixi-pack.json
unpack/
cache/
activate.*

# pixi environments
.pixi
*.egg-info
151 changes: 61 additions & 90 deletions .vscode/launch.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,93 +1,64 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'pixi-pack'",
"cargo": {
"args": [
"test",
"--no-run",
"--lib",
"--package=pixi-pack"
],
"filter": {
"name": "pixi-pack",
"kind": "lib"
}
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug 'pixi-pack pack'",
"cargo": {
"args": [
"build",
"--bin=pixi-pack",
"--package=pixi-pack"
],
"filter": {
"name": "pixi-pack",
"kind": "bin"
}
},
"args": [
"pack",
"-e",
"default",
"-p",
"osx-arm64",
"-m",
"test/pixi.toml"
],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug 'pixi-pack unpack'",
"cargo": {
"args": [
"build",
"--bin=pixi-pack",
"--package=pixi-pack"
],
"filter": {
"name": "pixi-pack",
"kind": "bin"
}
},
"args": [
"unpack",
"environment.tar.zstd"
],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in executable 'pixi-pack'",
"cargo": {
"args": [
"test",
"--no-run",
"--bin=pixi-pack",
"--package=pixi-pack"
],
"filter": {
"name": "pixi-pack",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}"
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in library 'pixi-pack'",
"cargo": {
"args": ["test", "--no-run", "--lib", "--package=pixi-pack"],
"filter": {
"name": "pixi-pack",
"kind": "lib"
}
]
},
"args": [],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug 'pixi-pack pack'",
"cargo": {
"args": ["build", "--bin=pixi-pack", "--package=pixi-pack"],
"filter": {
"name": "pixi-pack",
"kind": "bin"
}
},
"args": ["pack", "-e", "default", "-p", "osx-arm64", "-m", "test/pixi.toml"],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug 'pixi-pack unpack'",
"cargo": {
"args": ["build", "--bin=pixi-pack", "--package=pixi-pack"],
"filter": {
"name": "pixi-pack",
"kind": "bin"
}
},
"args": ["unpack", "environment.tar"],
"cwd": "${workspaceFolder}"
},
{
"type": "lldb",
"request": "launch",
"name": "Debug unit tests in executable 'pixi-pack'",
"cargo": {
"args": ["test", "--no-run", "--bin=pixi-pack", "--package=pixi-pack"],
"filter": {
"name": "pixi-pack",
"kind": "bin"
}
},
"args": [],
"cwd": "${workspaceFolder}"
}
]
}
3 changes: 0 additions & 3 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ tracing-subscriber = { version = "0.3.18", features = [
] }
tracing-log = "0.2"
url = "2.5.0"
async-compression = { version = "0.4.11", features = ["tokio", "zstd"] }
fxhash = "0.2.1"
tempfile = "3.10.1"

Expand Down
18 changes: 9 additions & 9 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ Starting with a [pixi](https://pixi.sh) lockfile `pixi.lock`, you can create a p
This environment can be unpacked on any system using `pixi-pack` to recreate the original environment.

In contrast to [`conda-pack`](https://conda.github.io/conda-pack/), `pixi-pack` does not require the original conda environment to be present on the system for packing.
Instead, it uses the lockfile to download the required packages and puts them into a `.tar.zstd` archive.
Instead, it uses the lockfile to download the required packages and puts them into a `.tar` archive.
This archive can then be shared with others and installed using `pixi-pack unpack` to recreate the original environment.

The original motivation behind `pixi-pack` was to create a `conda-pack` alternative that does not have the same reproducibility issues as `conda-pack`.
Expand All @@ -43,16 +43,16 @@ Or by downloading our pre-built binaries from the [releases page](https://github

### `pixi-pack pack`: Packing an environment

With `pixi-pack pack`, you can pack a conda environment into a `environment.tar.zstd` file:
With `pixi-pack pack`, you can pack a conda environment into a `environment.tar` file:

```bash
pixi-pack pack --manifest-file pixi.toml --environment prod --platform linux-64
```

This will create a `environment.tar.zstd` file that contains all conda packages required to create the environment.
This will create a `environment.tar` file that contains all conda packages required to create the environment.

```
# environment.tar.zstd
# environment.tar
| pixi-pack.json
| environment.yml
| channel
Expand All @@ -68,17 +68,17 @@ This will create a `environment.tar.zstd` file that contains all conda packages

### `pixi-pack unpack`: Unpacking an environment

With `pixi-pack unpack environment.tar.zstd`, you can unpack the environment on your target system.
With `pixi-pack unpack environment.tar`, you can unpack the environment on your target system.
This will create a new conda environment in `./env` that contains all packages specified in your `pixi.toml`.
It also creates an `activate.sh` (or `activate.bat` on Windows) file that lets you activate the environment
without needing to have `conda` or `micromamba` installed.

```bash
$ pixi-pack unpack environment.tar.zstd
$ pixi-pack unpack environment.tar
$ ls
env/
activate.sh
environment.tar.zstd
environment.tar
$ cat activate.sh
export PATH="/home/user/project/env/bin:..."
export CONDA_PREFIX="/home/user/project/env"
Expand Down Expand Up @@ -109,12 +109,12 @@ This can be particularly useful if you build the project itself and want to incl
### Unpacking without `pixi-pack`

If you don't have `pixi-pack` available on your target system, you can still install the environment if you have `conda` or `micromamba` available.
Just decompress the `environment.tar.zstd`, then you have a local channel on your system where all necessary packages are available.
Just unarchive the `environment.tar`, then you have a local channel on your system where all necessary packages are available.
Next to this local channel, you will find an `environment.yml` file that contains the environment specification.
You can then install the environment using `conda` or `micromamba`:

```bash
tar --zstd -xvf environment.tar.zstd
tar -xvf environment.tar
micromamba create -p ./env --file environment.yml
# or
conda env create -p ./env --file environment.yml
Expand Down
5 changes: 2 additions & 3 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ enum Commands {
#[arg(default_value = cwd().join("pixi.toml").into_os_string())]
manifest_path: PathBuf,

/// Output file to write the pack to
#[arg(short, long, default_value = cwd().join("environment.tar.zstd").into_os_string())]
/// Output file to write the pack to (will be an archive)
#[arg(short, long, default_value = cwd().join("environment.tar").into_os_string())]
output_file: PathBuf,

/// Inject an additional conda package into the final prefix
Expand Down Expand Up @@ -114,7 +114,6 @@ async fn main() -> Result<()> {
version: DEFAULT_PIXI_PACK_VERSION.to_string(),
platform,
},
level: None,
injected_packages: inject,
ignore_pypi_errors,
};
Expand Down
22 changes: 7 additions & 15 deletions src/pack.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ use tokio::{
};

use anyhow::Result;
use async_compression::{tokio::write::ZstdEncoder, Level};
use futures::{stream, StreamExt, TryFutureExt, TryStreamExt};
use indicatif::ProgressStyle;
use rattler_conda_types::{package::ArchiveType, ChannelInfo, PackageRecord, Platform, RepoData};
Expand All @@ -33,7 +32,6 @@ pub struct PackOptions {
pub output_file: PathBuf,
pub manifest_path: PathBuf,
pub metadata: PixiPackMetadata,
pub level: Option<Level>,
pub injected_packages: Vec<PathBuf>,
pub ignore_pypi_errors: bool,
}
Expand Down Expand Up @@ -171,8 +169,8 @@ pub async fn pack(options: PackOptions) -> Result<()> {
// Create environment file.
create_environment_file(output_folder.path(), conda_packages.iter().map(|(_, p)| p)).await?;

// Pack = archive + compress the contents.
archive_directory(output_folder.path(), &options.output_file, options.level)
// Pack = archive the contents.
archive_directory(output_folder.path(), &options.output_file)
.await
.map_err(|e| anyhow!("could not archive directory: {}", e))?;

Expand Down Expand Up @@ -232,12 +230,8 @@ async fn download_package(
Ok(())
}

/// Archive a directory into a compressed tarball.
async fn archive_directory(
input_dir: &Path,
archive_target: &Path,
level: Option<Level>,
) -> Result<()> {
/// Archive a directory into a tarball.
async fn archive_directory(input_dir: &Path, archive_target: &Path) -> Result<()> {
let outfile = fs::File::create(archive_target).await.map_err(|e| {
anyhow!(
"could not create archive file at {}: {}",
Expand All @@ -247,11 +241,7 @@ async fn archive_directory(
})?;

let writer = tokio::io::BufWriter::new(outfile);

let level = level.unwrap_or(Level::Default);
let compressor = ZstdEncoder::with_quality(writer, level);

let mut archive = Builder::new(compressor);
let mut archive = Builder::new(writer);

archive
.append_dir_all(".", input_dir)
Expand All @@ -271,6 +261,7 @@ async fn archive_directory(
Ok(())
}

/// Create an `environment.yml` file from the given packages.
async fn create_environment_file(
destination: &Path,
packages: impl IntoIterator<Item = &PackageRecord>,
Expand Down Expand Up @@ -302,6 +293,7 @@ async fn create_environment_file(
Ok(())
}

/// Create `repodata.json` files for the given packages.
async fn create_repodata_files(
packages: impl Iterator<Item = &(String, PackageRecord)>,
channel_dir: &Path,
Expand Down
8 changes: 2 additions & 6 deletions src/unpack.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use std::path::{Path, PathBuf};

use anyhow::{anyhow, Result};
use async_compression::tokio::bufread::ZstdDecoder;
use futures::{
stream::{self, StreamExt},
TryFutureExt, TryStreamExt,
Expand Down Expand Up @@ -130,17 +129,14 @@ async fn collect_packages(channel_dir: &Path) -> Result<FxHashMap<String, Packag
Ok(packages)
}

/// Unarchive a compressed tarball.
/// Unarchive a tarball.
pub async fn unarchive(archive_path: &Path, target_dir: &Path) -> Result<()> {
let file = fs::File::open(archive_path)
.await
.map_err(|e| anyhow!("could not open archive {:#?}: {}", archive_path, e))?;

let reader = tokio::io::BufReader::new(file);

let decoder = ZstdDecoder::new(reader);

let mut archive = Archive::new(decoder);
let mut archive = Archive::new(reader);

archive
.unpack(target_dir)
Expand Down
Loading
Loading