From bcf927f739452cd0a15225af36019c0d743cca1f Mon Sep 17 00:00:00 2001 From: Mathias-Boulay Date: Tue, 23 Apr 2024 09:55:34 +0200 Subject: [PATCH 01/25] fix(image-builder): use the proper tmp path Signed-off-by: Mathias-Boulay --- src/fs-gen/src/image_builder.rs | 24 +++++++++++++++--------- src/fs-gen/src/main.rs | 3 ++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/fs-gen/src/image_builder.rs b/src/fs-gen/src/image_builder.rs index 1d471f9..278a4ff 100644 --- a/src/fs-gen/src/image_builder.rs +++ b/src/fs-gen/src/image_builder.rs @@ -47,13 +47,18 @@ fn ensure_folder_created(output_folder: &Path) -> Result<()> { let result = fs::create_dir(output_folder); // If the file already exists, we're fine - if result.is_err() - && result - .unwrap_err() + if result.is_err() { + let err = result.unwrap_err(); + + if err .raw_os_error() .is_some_and(|err_val| err_val != FILE_EXISTS_ERROR) - { - return Err(anyhow!("Failed to create folder")); + { + return Err(anyhow!(err)).context(format!( + "Failed to create folder: {}", + output_folder.to_string_lossy() + )); + } } Ok(()) @@ -63,16 +68,17 @@ fn ensure_folder_created(output_folder: &Path) -> Result<()> { /// It works by instantiating an overlay fs via FUSE then copying the files to the desired target /// # Usage /// ``` -/// merge_layer(vec!["source/layer_1", "source/layer_2"], "/tmp/fused_layers") +/// merge_layer(vec!["source/layer_1", "source/layer_2"], "/tmp/fused_layers", "/tmp") /// ``` -pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path) -> Result<()> { +pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Path) -> Result<()> { // Stack all lower layers let mut lower_layers = Vec::new(); for lower in blob_paths { lower_layers.push(Arc::new(new_passthroughfs_layer(&lower.to_string_lossy())?)); } - let mountpoint = Path::new("/tmp/cloudlet_internal"); + let binding = tmp_folder.join("overlayfs_mountpoint"); + let mountpoint = binding.as_path(); let fs_name = "cloudlet_overlay"; ensure_folder_created(mountpoint)?; @@ -80,7 +86,7 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path) -> Result<()> { // Setup the overlay fs config let config = Config { - work: "/work".into(), + work: tmp_folder.join("work").to_string_lossy().into(), mountpoint: output_folder.to_string_lossy().into(), do_import: true, ..Default::default() diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 8229e85..618d0cd 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -30,7 +30,8 @@ fn main() { // FIXME: use a subdir of the temp directory instead let path = Path::new(overlay_subdir.as_path()); - merge_layer(&layers_paths, path).expect("Merging layers failed"); + merge_layer(&layers_paths, path, &args.temp_directory.clone()) + .expect("Merging layers failed"); create_init_file(path); generate_initramfs(path, Path::new(args.output_file.as_path())); } From 0b580e5f9d63e77ed5a5c7ca6afcd7eccf996f69 Mon Sep 17 00:00:00 2001 From: Mathias-Boulay Date: Tue, 23 Apr 2024 10:28:38 +0200 Subject: [PATCH 02/25] refactor: use a subfolder by default Signed-off-by: Mathias-Boulay --- src/fs-gen/src/image_builder.rs | 2 +- src/fs-gen/src/main.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fs-gen/src/image_builder.rs b/src/fs-gen/src/image_builder.rs index 278a4ff..4566f5a 100644 --- a/src/fs-gen/src/image_builder.rs +++ b/src/fs-gen/src/image_builder.rs @@ -44,7 +44,7 @@ fn new_passthroughfs_layer(rootdir: &str) -> Result { /// Ensure a destination folder is created fn ensure_folder_created(output_folder: &Path) -> Result<()> { - let result = fs::create_dir(output_folder); + let result = fs::create_dir_all(output_folder); // If the file already exists, we're fine if result.is_err() { diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 618d0cd..3a68fb0 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -13,6 +13,7 @@ fn main() { println!("Hello, world!, {:?}", args); let layers_subdir = args.temp_directory.clone().join("layers/"); + let output_subdir = args.temp_directory.clone().join("output/"); let overlay_subdir = args.temp_directory.clone().join("overlay/"); // TODO: better organise layers and OverlayFS build in the temp directory @@ -28,10 +29,9 @@ fn main() { } // FIXME: use a subdir of the temp directory instead - let path = Path::new(overlay_subdir.as_path()); + let path = Path::new(output_subdir.as_path()); - merge_layer(&layers_paths, path, &args.temp_directory.clone()) - .expect("Merging layers failed"); + merge_layer(&layers_paths, path, &overlay_subdir).expect("Merging layers failed"); create_init_file(path); generate_initramfs(path, Path::new(args.output_file.as_path())); } From 320bdfb57811771dd6bc1a0c1f5b725bbc4fb8ad Mon Sep 17 00:00:00 2001 From: Mathias Boulay Date: Tue, 23 Apr 2024 11:32:31 +0200 Subject: [PATCH 03/25] feat(arg-parsing): make the tag optional (#4) Signed-off-by: Mathias Boulay --- src/fs-gen/src/cli_args.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fs-gen/src/cli_args.rs b/src/fs-gen/src/cli_args.rs index dae89d6..a0ecd0c 100644 --- a/src/fs-gen/src/cli_args.rs +++ b/src/fs-gen/src/cli_args.rs @@ -7,7 +7,7 @@ use once_cell::sync::Lazy; // So, for any of you who may be scared, this is the regex from the OCI Distribution Sepcification for the image name + the tag static RE_IMAGE_NAME: Lazy = Lazy::new(|| { - Regex::new(r"[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}").unwrap() + Regex::new(r"[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*(?::[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})?").unwrap() }); /// Convert an OCI image into a CPIO file From e2c3e26c955d9b09bef1178f0d7e5634f25d9e1e Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:43:15 +0200 Subject: [PATCH 04/25] refactor: change default temp directory Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/cli_args.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/fs-gen/src/cli_args.rs b/src/fs-gen/src/cli_args.rs index a0ecd0c..46e15dd 100644 --- a/src/fs-gen/src/cli_args.rs +++ b/src/fs-gen/src/cli_args.rs @@ -66,13 +66,12 @@ impl CliArgs { } } -/// Get the default output path for the cpio file. +/// Get the default temporary directory for the current execution. fn get_default_temp_directory() -> PathBuf { - let mut path = env::current_dir().unwrap(); - path.push(".cloudlet_temp/"); - path + PathBuf::from("/tmp/cloudlet-fs-gen") } +/// Get the default output file path for the generated initramfs. fn get_default_output_file() -> PathBuf { let mut path = env::current_dir().unwrap(); path.push("initramfs.img"); From 526db41eb86b9d8c527e7ac50330c367bbe8b1c2 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:57:40 +0200 Subject: [PATCH 05/25] feat: insert initfile from command-line argument Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/cli_args.rs | 7 +++++-- src/fs-gen/src/initramfs_generator.rs | 26 ++++++++++++++++---------- src/fs-gen/src/main.rs | 2 +- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/fs-gen/src/cli_args.rs b/src/fs-gen/src/cli_args.rs index 46e15dd..2fbf138 100644 --- a/src/fs-gen/src/cli_args.rs +++ b/src/fs-gen/src/cli_args.rs @@ -17,6 +17,9 @@ pub struct CliArgs { /// The name of the image to download pub image_name: String, + /// The host path to the guest agent binary + pub agent_host_path: PathBuf, + /// The path to the output file #[arg(short='o', long="output", default_value=get_default_output_file().into_os_string())] pub output_file: PathBuf, @@ -25,8 +28,8 @@ pub struct CliArgs { #[arg(short='t', long="tempdir", default_value=get_default_temp_directory().into_os_string())] pub temp_directory: PathBuf, - /// The host path to the guest agent binary - pub agent_host_path: PathBuf, + #[arg(short='i', long="init", default_value=None)] + pub initfile_path: Option, } impl CliArgs { diff --git a/src/fs-gen/src/initramfs_generator.rs b/src/fs-gen/src/initramfs_generator.rs index 76027dc..12c5990 100644 --- a/src/fs-gen/src/initramfs_generator.rs +++ b/src/fs-gen/src/initramfs_generator.rs @@ -1,10 +1,10 @@ -use std::fs::{File, Permissions}; +use std::fs::{File, Permissions, copy}; use std::io::Write; use std::os::unix::fs::PermissionsExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -const INIT_FILE: &[u8; 211] = b"#! /bin/sh +const INIT_FILE: &[u8; 210] = b"#!/bin/sh # # Cloudlet initramfs generation # @@ -17,13 +17,19 @@ exec /sbin/getty -n -l /bin/sh 115200 /dev/console poweroff -f "; -pub fn create_init_file(path: &Path) { - let file_path = path.join("init"); - let mut file = File::create(file_path).unwrap(); - - file.write_all(INIT_FILE) - .expect("Could not write init file"); - file.set_permissions(Permissions::from_mode(0o755)).unwrap(); +pub fn create_init_file(path: &Path, initfile: Option) { + let destination = path.join("init"); + + if let Some(p) = initfile { + // if there is a given initfile, we copy it into the folder + copy(p, destination).expect("Could not copy initfile"); + } else { + // if there is none, write the default init file + let mut file = File::create(destination).unwrap(); + file.set_permissions(Permissions::from_mode(0o755)).unwrap(); + file.write_all(INIT_FILE) + .expect("Could not write init file"); + } } pub fn generate_initramfs(root_directory: &Path, output: &Path) { diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 3a68fb0..fecc4be 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -32,7 +32,7 @@ fn main() { let path = Path::new(output_subdir.as_path()); merge_layer(&layers_paths, path, &overlay_subdir).expect("Merging layers failed"); - create_init_file(path); + create_init_file(path, args.initfile_path); generate_initramfs(path, Path::new(args.output_file.as_path())); } } From eb4e6708585ddc1acf4e9c7de1c1aced3d05f5c0 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Tue, 23 Apr 2024 10:30:16 +0200 Subject: [PATCH 06/25] refactor: load initfile using include_str Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/resources/initfile | 11 +++++++++++ src/fs-gen/src/initramfs_generator.rs | 15 ++------------- 2 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 src/fs-gen/resources/initfile diff --git a/src/fs-gen/resources/initfile b/src/fs-gen/resources/initfile new file mode 100644 index 0000000..72144fb --- /dev/null +++ b/src/fs-gen/resources/initfile @@ -0,0 +1,11 @@ +#!/bin/sh +# +# Cloudlet initramfs generation +# +mount -t devtmpfs dev /dev +mount -t proc proc /proc +mount -t sysfs sysfs /sys +ip link set up dev lo + +exec /sbin/getty -n -l /bin/sh 115200 /dev/console +poweroff -f diff --git a/src/fs-gen/src/initramfs_generator.rs b/src/fs-gen/src/initramfs_generator.rs index 12c5990..d8e98a9 100644 --- a/src/fs-gen/src/initramfs_generator.rs +++ b/src/fs-gen/src/initramfs_generator.rs @@ -4,18 +4,7 @@ use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -const INIT_FILE: &[u8; 210] = b"#!/bin/sh -# -# Cloudlet initramfs generation -# -mount -t devtmpfs dev /dev -mount -t proc proc /proc -mount -t sysfs sysfs /sys -ip link set up dev lo - -exec /sbin/getty -n -l /bin/sh 115200 /dev/console -poweroff -f -"; +const INIT_FILE: &str = include_str!("../resources/initfile"); pub fn create_init_file(path: &Path, initfile: Option) { let destination = path.join("init"); @@ -27,7 +16,7 @@ pub fn create_init_file(path: &Path, initfile: Option) { // if there is none, write the default init file let mut file = File::create(destination).unwrap(); file.set_permissions(Permissions::from_mode(0o755)).unwrap(); - file.write_all(INIT_FILE) + file.write_all(INIT_FILE.as_bytes()) .expect("Could not write init file"); } } From 0b08f324ec51579121d3c0e9701d2e831dfe9bf3 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Tue, 23 Apr 2024 10:41:23 +0200 Subject: [PATCH 07/25] feat: insert agent when building initramfs Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/initramfs_generator.rs | 15 ++++++++++++--- src/fs-gen/src/main.rs | 4 +++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/fs-gen/src/initramfs_generator.rs b/src/fs-gen/src/initramfs_generator.rs index d8e98a9..9670ed5 100644 --- a/src/fs-gen/src/initramfs_generator.rs +++ b/src/fs-gen/src/initramfs_generator.rs @@ -1,8 +1,9 @@ -use std::fs::{File, Permissions, copy}; -use std::io::Write; +use std::fs::{File, Permissions, copy as fscopy}; +use std::io::{Write, copy as iocopy}; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use fuse_backend_rs::api::filesystem::ZeroCopyReader; const INIT_FILE: &str = include_str!("../resources/initfile"); @@ -11,7 +12,7 @@ pub fn create_init_file(path: &Path, initfile: Option) { if let Some(p) = initfile { // if there is a given initfile, we copy it into the folder - copy(p, destination).expect("Could not copy initfile"); + fscopy(p, destination).expect("Could not copy initfile"); } else { // if there is none, write the default init file let mut file = File::create(destination).unwrap(); @@ -21,6 +22,14 @@ pub fn create_init_file(path: &Path, initfile: Option) { } } +pub fn insert_agent(destination: &Path, agent_path: PathBuf) { + let mut file = File::create(destination.join("agent")).unwrap(); + file.set_permissions(Permissions::from_mode(0o755)).unwrap(); + + let mut agent = File::open(agent_path).unwrap(); + iocopy(&mut agent, &mut file).expect("Could not copy agent"); +} + pub fn generate_initramfs(root_directory: &Path, output: &Path) { let file = File::create(output).unwrap(); file.set_permissions(Permissions::from_mode(0o644)) diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index fecc4be..125b48f 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,6 +1,6 @@ use std::{fs::remove_dir_all, path::Path}; -use crate::initramfs_generator::{create_init_file, generate_initramfs}; +use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use image_builder::merge_layer; mod cli_args; @@ -33,6 +33,8 @@ fn main() { merge_layer(&layers_paths, path, &overlay_subdir).expect("Merging layers failed"); create_init_file(path, args.initfile_path); + insert_agent(path, args.agent_host_path); + generate_initramfs(path, Path::new(args.output_file.as_path())); } } From d962d89c34bf1ca12ae134797efe8b1a6105b388 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Tue, 23 Apr 2024 11:21:20 +0200 Subject: [PATCH 08/25] chore: add tracing crate Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fs-gen/Cargo.toml b/src/fs-gen/Cargo.toml index 6bccfdb..191df67 100644 --- a/src/fs-gen/Cargo.toml +++ b/src/fs-gen/Cargo.toml @@ -19,3 +19,4 @@ signal-hook = "0.3.17" tar = "0.4.40" validator = { version = "0.17.0", features = ["derive"] } anyhow = "1.0.82" +tracing = "0.1.40" From 88412a194e7c35f8d8d09d26af33f0a5bbb9d955 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:02:06 +0200 Subject: [PATCH 09/25] feat: implement tracing for crate Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/Cargo.toml | 1 + src/fs-gen/src/cli_args.rs | 5 ++++- src/fs-gen/src/image_builder.rs | 12 +++++------ src/fs-gen/src/image_loader.rs | 21 +++++++++++++------ src/fs-gen/src/initramfs_generator.rs | 14 ++++++++++--- src/fs-gen/src/main.rs | 30 ++++++++++++++++++++------- 6 files changed, 59 insertions(+), 24 deletions(-) diff --git a/src/fs-gen/Cargo.toml b/src/fs-gen/Cargo.toml index 191df67..8f43c41 100644 --- a/src/fs-gen/Cargo.toml +++ b/src/fs-gen/Cargo.toml @@ -20,3 +20,4 @@ tar = "0.4.40" validator = { version = "0.17.0", features = ["derive"] } anyhow = "1.0.82" tracing = "0.1.40" +tracing-subscriber = "0.3.18" diff --git a/src/fs-gen/src/cli_args.rs b/src/fs-gen/src/cli_args.rs index 2fbf138..2a20de9 100644 --- a/src/fs-gen/src/cli_args.rs +++ b/src/fs-gen/src/cli_args.rs @@ -1,6 +1,6 @@ use std::{env, path::PathBuf}; -use clap::{command, error::ErrorKind, CommandFactory, Parser}; +use clap::{command, error::ErrorKind, CommandFactory, Parser, ArgAction}; use regex::Regex; use once_cell::sync::Lazy; @@ -30,6 +30,9 @@ pub struct CliArgs { #[arg(short='i', long="init", default_value=None)] pub initfile_path: Option, + + #[arg(short='d', long="debug", action=ArgAction::SetTrue)] + pub debug: bool, } impl CliArgs { diff --git a/src/fs-gen/src/image_builder.rs b/src/fs-gen/src/image_builder.rs index 4566f5a..026b990 100644 --- a/src/fs-gen/src/image_builder.rs +++ b/src/fs-gen/src/image_builder.rs @@ -13,6 +13,7 @@ use fuse_backend_rs::{ passthrough::{self, PassthroughFs}, transport::{FuseChannel, FuseSession}, }; +use tracing::{debug, warn}; static FILE_EXISTS_ERROR: i32 = 17; @@ -116,7 +117,7 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Pa let _ = server.svc_loop(); }); - println!("copy starting !"); + debug!("Starting copy..."); //So now we need to copy the files dircpy::copy_dir(mountpoint, output_folder).with_context(|| { format!( @@ -124,7 +125,7 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Pa output_folder.to_string_lossy() ) })?; - println!("copy finished"); + debug!("Copy finished!"); // Unmount sessions so it can be re-used in later executions of the program se.wake() @@ -138,9 +139,8 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Pa impl FuseServer { /// Run a loop to execute requests from the FUSE session - /// pub fn svc_loop(&mut self) -> Result<()> { - println!("entering server loop"); + debug!("entering server loop"); loop { let value = self .ch @@ -148,7 +148,7 @@ impl FuseServer { .with_context(|| "Failed to get message from fuse session".to_string())?; if value.is_none() { - println!("fuse server exits"); + debug!("fuse server exits"); break; } @@ -164,7 +164,7 @@ impl FuseServer { break; } _ => { - print!("Handling fuse message failed"); + warn!("Handling fuse message failed"); continue; } } diff --git a/src/fs-gen/src/image_loader.rs b/src/fs-gen/src/image_loader.rs index 45c77bb..3bbf48a 100644 --- a/src/fs-gen/src/image_loader.rs +++ b/src/fs-gen/src/image_loader.rs @@ -4,6 +4,7 @@ use std::error::Error; use std::fs::create_dir; use std::path::PathBuf; use tar::Archive; +use tracing::{debug, info}; pub fn download_image_fs( image_name: &str, @@ -33,7 +34,8 @@ pub fn download_image_fs( image_name, tag ))?, Some(m) => { - println!("Manifest list found. Looking for an amd64 manifest..."); + debug!("Manifest list found. Looking for an amd64 manifest..."); + // TODO: implement other than amd64? // Get a manifest for amd64 architecture from the manifest list let amd64_manifest = m.iter().find(|manifest| { manifest["platform"].as_object().unwrap()["architecture"] @@ -45,7 +47,8 @@ pub fn download_image_fs( match amd64_manifest { None => Err("This image doesn't support amd64 architecture")?, Some(m) => { - println!("Downloading manifest for amd64 architecture..."); + info!("Downloading image..."); + debug!("Downloading manifest for amd64 architecture..."); manifest_json = download_manifest(image_name, m["digest"].as_str().unwrap())?; layers = manifest_json["layers"].as_array(); @@ -96,7 +99,10 @@ fn download_manifest(image_name: &str, digest: &str) -> Result) { + info!("Writing initfile..."); + let destination = path.join("init"); if let Some(p) = initfile { @@ -20,14 +22,20 @@ pub fn create_init_file(path: &Path, initfile: Option) { file.write_all(INIT_FILE.as_bytes()) .expect("Could not write init file"); } + + info!("Initfile written!"); } pub fn insert_agent(destination: &Path, agent_path: PathBuf) { + info!("Inserting agent into fs..."); + let mut file = File::create(destination.join("agent")).unwrap(); file.set_permissions(Permissions::from_mode(0o755)).unwrap(); let mut agent = File::open(agent_path).unwrap(); iocopy(&mut agent, &mut file).expect("Could not copy agent"); + + info!("Agent inserted!"); } pub fn generate_initramfs(root_directory: &Path, output: &Path) { @@ -35,7 +43,7 @@ pub fn generate_initramfs(root_directory: &Path, output: &Path) { file.set_permissions(Permissions::from_mode(0o644)) .expect("Could not set permissions"); - println!("Generating initramfs..."); + info!("Generating initramfs..."); let mut command = Command::new("sh") .current_dir(root_directory) @@ -48,5 +56,5 @@ pub fn generate_initramfs(root_directory: &Path, output: &Path) { .wait() .expect("Failed to wait for initramfs to finish"); - println!("Initramfs generated!"); + info!("Initramfs generated!"); } diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 125b48f..c9a9634 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,4 +1,6 @@ use std::{fs::remove_dir_all, path::Path}; +use tracing::{debug, error, info, Level}; +use tracing_subscriber; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use image_builder::merge_layer; @@ -10,28 +12,40 @@ mod initramfs_generator; fn main() { let args = cli_args::CliArgs::get_args(); - println!("Hello, world!, {:?}", args); + + tracing_subscriber::fmt() + .with_max_level(if args.debug { Level::DEBUG } else { Level::INFO }) + .init(); + + info!("Cloudlet initramfs generator v{}", env!("CARGO_PKG_VERSION")); + info!("Generating for image '{}'", args.image_name); + + debug!( + image_name = args.image_name, + agent_host_path = ?args.agent_host_path, + output_file = ?args.output_file, + temp_dir = ?args.temp_directory, + initfile_path = ?args.initfile_path, + debug = args.debug, + "arguments:", + ); let layers_subdir = args.temp_directory.clone().join("layers/"); let output_subdir = args.temp_directory.clone().join("output/"); let overlay_subdir = args.temp_directory.clone().join("overlay/"); - // TODO: better organise layers and OverlayFS build in the temp directory match image_loader::download_image_fs(&args.image_name, layers_subdir) { Err(e) => { - eprintln!("Error: {}", e); + error!(%e, "Received error while downloading image"); return; } Ok(layers_paths) => { - println!("Image downloaded successfully! Layers' paths:"); - for path in &layers_paths { - println!(" - {}", path.display()); - } + debug!("Layers' paths: {:?}", layers_paths); - // FIXME: use a subdir of the temp directory instead let path = Path::new(output_subdir.as_path()); merge_layer(&layers_paths, path, &overlay_subdir).expect("Merging layers failed"); + create_init_file(path, args.initfile_path); insert_agent(path, args.agent_host_path); From 9d943d3cab0af72ff36ee0201e93b69a6f8e4452 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Wed, 24 Apr 2024 11:03:56 +0200 Subject: [PATCH 10/25] lint: remove unused import Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index c9a9634..410f181 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,6 +1,5 @@ use std::{fs::remove_dir_all, path::Path}; use tracing::{debug, error, info, Level}; -use tracing_subscriber; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use image_builder::merge_layer; From 5a1dc2492ae16eae69cd52caa20bff611394e036 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Wed, 24 Apr 2024 09:35:19 +0200 Subject: [PATCH 11/25] refactor: implement better error handling for initramfs generation Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/initramfs_generator.rs | 41 ++++++++++++++++++--------- src/fs-gen/src/main.rs | 7 ++--- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/fs-gen/src/initramfs_generator.rs b/src/fs-gen/src/initramfs_generator.rs index df98e15..f9c5d40 100644 --- a/src/fs-gen/src/initramfs_generator.rs +++ b/src/fs-gen/src/initramfs_generator.rs @@ -4,44 +4,56 @@ use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use tracing::info; +use anyhow::{Context, Result}; const INIT_FILE: &str = include_str!("../resources/initfile"); -pub fn create_init_file(path: &Path, initfile: Option) { +pub fn create_init_file(path: &Path, initfile: Option) -> Result<()> { info!("Writing initfile..."); let destination = path.join("init"); if let Some(p) = initfile { // if there is a given initfile, we copy it into the folder - fscopy(p, destination).expect("Could not copy initfile"); + fscopy(p, destination) + .with_context(|| "Failed to copy provided initfile to initramfs".to_string())?; } else { // if there is none, write the default init file let mut file = File::create(destination).unwrap(); file.set_permissions(Permissions::from_mode(0o755)).unwrap(); + file.write_all(INIT_FILE.as_bytes()) - .expect("Could not write init file"); + .with_context(|| "Failed to write default initfile to initramfs".to_string())?; } info!("Initfile written!"); + + Ok(()) } -pub fn insert_agent(destination: &Path, agent_path: PathBuf) { +pub fn insert_agent(destination: &Path, agent_path: PathBuf) -> Result<()> { info!("Inserting agent into fs..."); - let mut file = File::create(destination.join("agent")).unwrap(); - file.set_permissions(Permissions::from_mode(0o755)).unwrap(); + let mut file = File::create(destination.join("agent")) + .with_context(|| "Could not open agent file inside initramfs".to_string())?; + file.set_permissions(Permissions::from_mode(0o755)) + .with_context(|| "Failed to set permissions for agent file".to_string())?; - let mut agent = File::open(agent_path).unwrap(); - iocopy(&mut agent, &mut file).expect("Could not copy agent"); + let mut agent = File::open(agent_path) + .with_context(|| "Could not open host agent file".to_string())?; + iocopy(&mut agent, &mut file) + .with_context(|| "Failed to copy agent contents from host to destination".to_string())?; info!("Agent inserted!"); + + Ok(()) } -pub fn generate_initramfs(root_directory: &Path, output: &Path) { - let file = File::create(output).unwrap(); +pub fn generate_initramfs(root_directory: &Path, output: &Path) -> Result<()> { + let file = File::create(output) + .with_context(|| "Could not open output file to write initramfs".to_string())?; file.set_permissions(Permissions::from_mode(0o644)) - .expect("Could not set permissions"); + .with_context(|| "Failed to set permissions for output file".to_string())?; info!("Generating initramfs..."); @@ -51,10 +63,13 @@ pub fn generate_initramfs(root_directory: &Path, output: &Path) { .arg("-c") .arg("find . -print0 | cpio -0 --create --owner=root:root --format=newc | xz -9 --format=lzma") .spawn() - .expect("Failed to package initramfs"); + .with_context(|| "Failed to package initramfs into bundle".to_string())?; + command .wait() - .expect("Failed to wait for initramfs to finish"); + .with_context(|| "Encountered exception while waiting for bundling to finish".to_string())?; info!("Initramfs generated!"); + + Ok(()) } diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 410f181..87fbf29 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -44,11 +44,10 @@ fn main() { let path = Path::new(output_subdir.as_path()); merge_layer(&layers_paths, path, &overlay_subdir).expect("Merging layers failed"); + create_init_file(path, args.initfile_path)?; + insert_agent(path, args.agent_host_path)?; - create_init_file(path, args.initfile_path); - insert_agent(path, args.agent_host_path); - - generate_initramfs(path, Path::new(args.output_file.as_path())); + generate_initramfs(path, Path::new(args.output_file.as_path()))?; } } From 60c5b047fa2f87e62043e14bb50119f06e406808 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:17:23 +0200 Subject: [PATCH 12/25] refactor: add error handling for image loading Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/Cargo.toml | 1 + src/fs-gen/src/errors.rs | 23 +++++++++++++ src/fs-gen/src/image_loader.rs | 60 ++++++++++++++++++---------------- src/fs-gen/src/main.rs | 18 ++++++---- 4 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 src/fs-gen/src/errors.rs diff --git a/src/fs-gen/Cargo.toml b/src/fs-gen/Cargo.toml index 8f43c41..a131d9e 100644 --- a/src/fs-gen/Cargo.toml +++ b/src/fs-gen/Cargo.toml @@ -21,3 +21,4 @@ validator = { version = "0.17.0", features = ["derive"] } anyhow = "1.0.82" tracing = "0.1.40" tracing-subscriber = "0.3.18" +thiserror = "1.0.59" diff --git a/src/fs-gen/src/errors.rs b/src/fs-gen/src/errors.rs new file mode 100644 index 0000000..e892904 --- /dev/null +++ b/src/fs-gen/src/errors.rs @@ -0,0 +1,23 @@ + +use thiserror::Error; + +#[derive(Debug, Error)] +pub(crate) enum ImageLoaderError { + /// There is no existing manifest for the given image. + #[error("Could not find Docker v2 or OCI manifest for `{0}:{1}`")] + ManifestNotFound(String, String), + + /// Image doesn't support the requested architecture. + #[error("This image doesn't support {0} architecture")] + UnsupportedArchitecture(String), + + /// The manifest doesn't contain any layers to unpack. + #[error("Could not find image layers in the manifest")] + LayersNotFound, + + /// Encountered an error during the flow. + #[error("Image loading error: {}", .source)] + Error { + source: anyhow::Error + } +} diff --git a/src/fs-gen/src/image_loader.rs b/src/fs-gen/src/image_loader.rs index 3bbf48a..6e72dee 100644 --- a/src/fs-gen/src/image_loader.rs +++ b/src/fs-gen/src/image_loader.rs @@ -1,15 +1,16 @@ use flate2::read::GzDecoder; use reqwest::blocking::{Client, Response}; -use std::error::Error; use std::fs::create_dir; use std::path::PathBuf; use tar::Archive; use tracing::{debug, info}; +use anyhow::{Context, Result}; +use crate::errors::ImageLoaderError; pub fn download_image_fs( image_name: &str, output_file: PathBuf, -) -> Result, Box> { +) -> Result, ImageLoaderError> { // Get image's name and tag let image_and_tag: Vec<&str> = image_name.split(':').collect(); @@ -21,7 +22,8 @@ pub fn download_image_fs( let image_name = image_and_tag[0]; // Download image manifest - let mut manifest_json = download_manifest(image_name, tag)?; + let mut manifest_json = download_manifest(image_name, tag) + .map_err(|e| ImageLoaderError::Error { source: e })?; // Verify if it's a manifest or a manifest list let mut layers = manifest_json["layers"].as_array(); @@ -29,10 +31,7 @@ pub fn download_image_fs( if layers.is_none() { let manifests = manifest_json["manifests"].as_array(); match manifests { - None => Err(format!( - "Couldn't find a Docker V2 or OCI manifest for {}:{}", - image_name, tag - ))?, + None => Err(ImageLoaderError::ManifestNotFound(image_name.to_string(), tag.to_string()))?, Some(m) => { debug!("Manifest list found. Looking for an amd64 manifest..."); // TODO: implement other than amd64? @@ -45,15 +44,16 @@ pub fn download_image_fs( }); match amd64_manifest { - None => Err("This image doesn't support amd64 architecture")?, + None => Err(ImageLoaderError::UnsupportedArchitecture("amd64".to_string()))?, Some(m) => { info!("Downloading image..."); debug!("Downloading manifest for amd64 architecture..."); manifest_json = - download_manifest(image_name, m["digest"].as_str().unwrap())?; + download_manifest(image_name, m["digest"].as_str().unwrap()) + .map_err(|e| ImageLoaderError::Error { source: e })?; layers = manifest_json["layers"].as_array(); if layers.is_none() { - Err("Couldn't find image layers in the manifest.")? + Err(ImageLoaderError::LayersNotFound)? } } } @@ -63,19 +63,20 @@ pub fn download_image_fs( let _ = create_dir(&output_file); - download_layers(layers.unwrap(), image_name, &output_file) + download_layers(layers.unwrap(), image_name, &output_file).map_err(|e| ImageLoaderError::Error { source: e }) } -fn download_manifest(image_name: &str, digest: &str) -> Result> { +fn download_manifest(image_name: &str, digest: &str) -> Result { // Create a reqwest HTTP client let client = Client::new(); // Get a token for anonymous authentication to Docker Hub let token_json: serde_json::Value = client .get(format!("https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/{image_name}:pull")) - .send()?.json()?; + .send().with_context(|| "Could not send request for anonymous authentication".to_string())? + .json().with_context(|| "Failed to parse JSON response for anonymous authentication".to_string())?; - let token = token_json["token"].as_str().unwrap(); + let token = token_json["token"].as_str().with_context(|| "Failed to get token from anon auth response".to_string())?; // Query Docker Hub API to get the image manifest let manifest_url = format!( @@ -83,7 +84,7 @@ fn download_manifest(image_name: &str, digest: &str) -> Result Result, output_dir: &PathBuf) -> Result<(), Box> { - let mut ar = Archive::new(tar); - ar.unpack(output_dir)?; +fn unpack_tarball(tar: GzDecoder, output_dir: &PathBuf) -> Result<()> { + Archive::new(tar).unpack(output_dir.clone()) + .with_context(|| format!("Failed to unpack tarball to {}", output_dir.display()))?; Ok(()) } @@ -117,15 +117,17 @@ fn download_layers( layers: &Vec, image_name: &str, output_dir: &PathBuf, -) -> Result, Box> { +) -> Result> { let client = Client::new(); // Get a token for anonymous authentication to Docker Hub let token_json: serde_json::Value = client .get(format!("https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/{image_name}:pull")) - .send()?.json()?; + .send().with_context(|| "Could not send request for anon authentication (layers)".to_string())? + .json().with_context(|| "Failed to parse JSON response for anonymous authentication (layers)".to_string())?; - let token = token_json["token"].as_str().unwrap(); + let token = token_json["token"].as_str() + .with_context(|| "Failed to get token from anon auth response (layers)".to_string())?; let mut layer_paths = Vec::new(); @@ -133,13 +135,15 @@ fn download_layers( // Download and unpack each layer for layer in layers { - let digest = layer["digest"].as_str().unwrap(); + let digest = layer["digest"].as_str() + .with_context(|| "Failed to get digest for layer".to_string())?; let layer_url = format!( "https://registry-1.docker.io/v2/library/{}/blobs/{}", image_name, digest ); - let response = client.get(&layer_url).bearer_auth(token).send()?; + let response = client.get(&layer_url).bearer_auth(token) + .send().with_context(|| format!("Could not send request for layer digest '{digest}'"))?; debug!("starting to decode layer with digest '{}'", digest); diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 87fbf29..f149885 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,15 +1,17 @@ use std::{fs::remove_dir_all, path::Path}; use tracing::{debug, error, info, Level}; +use anyhow::{Result, Error, bail}; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; -use image_builder::merge_layer; +use crate::image_builder::merge_layer; mod cli_args; mod image_builder; mod image_loader; mod initramfs_generator; +mod errors; -fn main() { +fn main() -> Result<()> { let args = cli_args::CliArgs::get_args(); tracing_subscriber::fmt() @@ -35,8 +37,8 @@ fn main() { match image_loader::download_image_fs(&args.image_name, layers_subdir) { Err(e) => { - error!(%e, "Received error while downloading image"); - return; + error!(error = ?e, "image loader error"); + bail!(e) } Ok(layers_paths) => { debug!("Layers' paths: {:?}", layers_paths); @@ -48,9 +50,11 @@ fn main() { insert_agent(path, args.agent_host_path)?; generate_initramfs(path, Path::new(args.output_file.as_path()))?; + + // cleanup of temporary directory + remove_dir_all(args.temp_directory.clone()).expect("Could not remove temporary directory"); + + Ok(()) } } - - // cleanup of temporary directory - remove_dir_all(args.temp_directory.clone()).expect("Could not remove temporary directory"); } From b0e4143899978fa897fe994a84bbf888697b778f Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:07:50 +0200 Subject: [PATCH 13/25] refactor: log errors for each soft unwrap Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/main.rs | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index f149885..7d0c349 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,6 +1,6 @@ use std::{fs::remove_dir_all, path::Path}; use tracing::{debug, error, info, Level}; -use anyhow::{Result, Error, bail}; +use anyhow::{Result, Error, bail, Context}; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use crate::image_builder::merge_layer; @@ -46,13 +46,28 @@ fn main() -> Result<()> { let path = Path::new(output_subdir.as_path()); merge_layer(&layers_paths, path, &overlay_subdir).expect("Merging layers failed"); - create_init_file(path, args.initfile_path)?; - insert_agent(path, args.agent_host_path)?; - generate_initramfs(path, Path::new(args.output_file.as_path()))?; + if let Err(e) = create_init_file(path, args.initfile_path) { + error!(error = ?e, "while creating init file"); + bail!(e) + } + + if let Err(e) = insert_agent(path, args.agent_host_path) { + error!(error = ?e, "while inserting agent"); + bail!(e) + } + + if let Err(e) = generate_initramfs(path, Path::new(args.output_file.as_path())) { + error!(error = ?e, "while generating initramfs"); + bail!(e) + } // cleanup of temporary directory - remove_dir_all(args.temp_directory.clone()).expect("Could not remove temporary directory"); + if let Err(e) = remove_dir_all(args.temp_directory.clone()) + .with_context(|| "Failed to remove temporary directory".to_string()) { + error!(?e, ""); + bail!(e) + } Ok(()) } From 1c1ac095c8d26f7f39533438a7603eac9d7fdc67 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:16:46 +0200 Subject: [PATCH 14/25] refactor: simplify error handling in main function Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/main.rs | 82 +++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 7d0c349..3990f5c 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,6 +1,8 @@ use std::{fs::remove_dir_all, path::Path}; +use std::path::PathBuf; use tracing::{debug, error, info, Level}; use anyhow::{Result, Error, bail, Context}; +use crate::cli_args::CliArgs; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use crate::image_builder::merge_layer; @@ -11,8 +13,35 @@ mod image_loader; mod initramfs_generator; mod errors; +fn run( + args: CliArgs, + layers_subdir: PathBuf, + output_subdir: PathBuf, + overlay_subdir: PathBuf, +) -> Result<()> { + let path = Path::new(output_subdir.as_path()); + + // image downloading and unpacking + let layers_paths = image_loader::download_image_fs(&args.image_name, layers_subdir)?; + debug!("Layers' paths: {:?}", layers_paths); + + // reconstructing image with overlayfs + merge_layer(&layers_paths, path, &overlay_subdir)?; + + // building initramfs + create_init_file(path, args.initfile_path)?; + insert_agent(path, args.agent_host_path)?; + generate_initramfs(path, Path::new(args.output_file.as_path()))?; + + // cleanup of temporary directory + remove_dir_all(args.temp_directory.clone()) + .with_context(|| "Failed to remove temporary directory".to_string())?; + + Ok(()) +} + fn main() -> Result<()> { - let args = cli_args::CliArgs::get_args(); + let args = CliArgs::get_args(); tracing_subscriber::fmt() .with_max_level(if args.debug { Level::DEBUG } else { Level::INFO }) @@ -31,45 +60,16 @@ fn main() -> Result<()> { "arguments:", ); - let layers_subdir = args.temp_directory.clone().join("layers/"); - let output_subdir = args.temp_directory.clone().join("output/"); - let overlay_subdir = args.temp_directory.clone().join("overlay/"); - - match image_loader::download_image_fs(&args.image_name, layers_subdir) { - Err(e) => { - error!(error = ?e, "image loader error"); - bail!(e) - } - Ok(layers_paths) => { - debug!("Layers' paths: {:?}", layers_paths); - - let path = Path::new(output_subdir.as_path()); - - merge_layer(&layers_paths, path, &overlay_subdir).expect("Merging layers failed"); - - if let Err(e) = create_init_file(path, args.initfile_path) { - error!(error = ?e, "while creating init file"); - bail!(e) - } - - if let Err(e) = insert_agent(path, args.agent_host_path) { - error!(error = ?e, "while inserting agent"); - bail!(e) - } - - if let Err(e) = generate_initramfs(path, Path::new(args.output_file.as_path())) { - error!(error = ?e, "while generating initramfs"); - bail!(e) - } - - // cleanup of temporary directory - if let Err(e) = remove_dir_all(args.temp_directory.clone()) - .with_context(|| "Failed to remove temporary directory".to_string()) { - error!(?e, ""); - bail!(e) - } - - Ok(()) - } + if let Err(e) = run( + args, + args.temp_directory.clone().join("layers/"), + args.temp_directory.clone().join("output/"), + args.temp_directory.clone().join("overlay/") + ) { + error!(error = ?e, "encountered error while running"); + Err(e) + } else { + info!("Finished successfully!"); + Ok(()) } } From e6bf37efdd24043bc30bef4c421e4182f0263f08 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:14:09 +0200 Subject: [PATCH 15/25] refactor: rework architecture for downloading and unpacking Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/cli_args.rs | 2 +- .../{image_loader.rs => loader/download.rs} | 62 ++++++++----------- src/fs-gen/src/{ => loader}/errors.rs | 8 ++- src/fs-gen/src/loader/mod.rs | 3 + src/fs-gen/src/loader/utils.rs | 25 ++++++++ src/fs-gen/src/main.rs | 17 ++--- 6 files changed, 72 insertions(+), 45 deletions(-) rename src/fs-gen/src/{image_loader.rs => loader/download.rs} (67%) rename src/fs-gen/src/{ => loader}/errors.rs (82%) create mode 100644 src/fs-gen/src/loader/mod.rs create mode 100644 src/fs-gen/src/loader/utils.rs diff --git a/src/fs-gen/src/cli_args.rs b/src/fs-gen/src/cli_args.rs index 2a20de9..011406a 100644 --- a/src/fs-gen/src/cli_args.rs +++ b/src/fs-gen/src/cli_args.rs @@ -11,7 +11,7 @@ static RE_IMAGE_NAME: Lazy = Lazy::new(|| { }); /// Convert an OCI image into a CPIO file -#[derive(Parser, Debug)] +#[derive(Parser, Debug, Clone)] #[command(version, about, long_about = None)] pub struct CliArgs { /// The name of the image to download diff --git a/src/fs-gen/src/image_loader.rs b/src/fs-gen/src/loader/download.rs similarity index 67% rename from src/fs-gen/src/image_loader.rs rename to src/fs-gen/src/loader/download.rs index 6e72dee..f2f6e44 100644 --- a/src/fs-gen/src/image_loader.rs +++ b/src/fs-gen/src/loader/download.rs @@ -5,9 +5,10 @@ use std::path::PathBuf; use tar::Archive; use tracing::{debug, info}; use anyhow::{Context, Result}; -use crate::errors::ImageLoaderError; +use crate::loader::errors::ImageLoaderError; +use crate::loader::utils::{get_docker_download_token, unpack_tarball}; -pub fn download_image_fs( +pub(crate) fn download_image_fs( image_name: &str, output_file: PathBuf, ) -> Result, ImageLoaderError> { @@ -21,8 +22,12 @@ pub fn download_image_fs( }; let image_name = image_and_tag[0]; + // Get download token + let client = Client::new(); + let token = &get_docker_download_token(&client, image_name)?; + // Download image manifest - let mut manifest_json = download_manifest(image_name, tag) + let mut manifest_json = download_manifest(&client, token, image_name, tag) .map_err(|e| ImageLoaderError::Error { source: e })?; // Verify if it's a manifest or a manifest list @@ -48,9 +53,14 @@ pub fn download_image_fs( Some(m) => { info!("Downloading image..."); debug!("Downloading manifest for amd64 architecture..."); - manifest_json = - download_manifest(image_name, m["digest"].as_str().unwrap()) - .map_err(|e| ImageLoaderError::Error { source: e })?; + + manifest_json = download_manifest( + &client, + token, + image_name, + m["digest"].as_str().unwrap() + ).map_err(|e| ImageLoaderError::Error { source: e })?; + layers = manifest_json["layers"].as_array(); if layers.is_none() { Err(ImageLoaderError::LayersNotFound)? @@ -63,21 +73,16 @@ pub fn download_image_fs( let _ = create_dir(&output_file); - download_layers(layers.unwrap(), image_name, &output_file).map_err(|e| ImageLoaderError::Error { source: e }) + download_layers( + layers.unwrap(), + &client, + token, + image_name, + &output_file + ).map_err(|e| ImageLoaderError::Error { source: e }) } -fn download_manifest(image_name: &str, digest: &str) -> Result { - // Create a reqwest HTTP client - let client = Client::new(); - - // Get a token for anonymous authentication to Docker Hub - let token_json: serde_json::Value = client - .get(format!("https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/{image_name}:pull")) - .send().with_context(|| "Could not send request for anonymous authentication".to_string())? - .json().with_context(|| "Failed to parse JSON response for anonymous authentication".to_string())?; - - let token = token_json["token"].as_str().with_context(|| "Failed to get token from anon auth response".to_string())?; - +fn download_manifest(client: &Client, token: &str, image_name: &str, digest: &str) -> Result { // Query Docker Hub API to get the image manifest let manifest_url = format!( "https://registry-1.docker.io/v2/library/{}/manifests/{}", @@ -107,28 +112,13 @@ fn download_manifest(image_name: &str, digest: &str) -> Result, output_dir: &PathBuf) -> Result<()> { - Archive::new(tar).unpack(output_dir.clone()) - .with_context(|| format!("Failed to unpack tarball to {}", output_dir.display()))?; - Ok(()) -} - fn download_layers( layers: &Vec, + client: &Client, + token: &str, image_name: &str, output_dir: &PathBuf, ) -> Result> { - let client = Client::new(); - - // Get a token for anonymous authentication to Docker Hub - let token_json: serde_json::Value = client - .get(format!("https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/{image_name}:pull")) - .send().with_context(|| "Could not send request for anon authentication (layers)".to_string())? - .json().with_context(|| "Failed to parse JSON response for anonymous authentication (layers)".to_string())?; - - let token = token_json["token"].as_str() - .with_context(|| "Failed to get token from anon auth response (layers)".to_string())?; - let mut layer_paths = Vec::new(); debug!("Downloading and unpacking layers..."); diff --git a/src/fs-gen/src/errors.rs b/src/fs-gen/src/loader/errors.rs similarity index 82% rename from src/fs-gen/src/errors.rs rename to src/fs-gen/src/loader/errors.rs index e892904..b6fda91 100644 --- a/src/fs-gen/src/errors.rs +++ b/src/fs-gen/src/loader/errors.rs @@ -1,4 +1,4 @@ - +use anyhow::Error; use thiserror::Error; #[derive(Debug, Error)] @@ -21,3 +21,9 @@ pub(crate) enum ImageLoaderError { source: anyhow::Error } } + +impl From for ImageLoaderError { + fn from(value: Error) -> Self { + Self::Error { source: value } + } +} diff --git a/src/fs-gen/src/loader/mod.rs b/src/fs-gen/src/loader/mod.rs new file mode 100644 index 0000000..866053d --- /dev/null +++ b/src/fs-gen/src/loader/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod download; +pub(crate) mod errors; +pub(self) mod utils; diff --git a/src/fs-gen/src/loader/utils.rs b/src/fs-gen/src/loader/utils.rs new file mode 100644 index 0000000..98ab949 --- /dev/null +++ b/src/fs-gen/src/loader/utils.rs @@ -0,0 +1,25 @@ +use flate2::read::GzDecoder; +use reqwest::blocking::{Client, Response}; +use std::path::PathBuf; +use tar::Archive; +use anyhow::{Context, Result}; + +/// Unpack the tarball to a given directory. +pub(super) fn unpack_tarball(tar: GzDecoder, output_dir: &PathBuf) -> Result<()> { + Archive::new(tar).unpack(output_dir.clone()) + .with_context(|| format!("Failed to unpack tarball to {}", output_dir.display()))?; + Ok(()) +} + +/// Get a token for anonymous authentication to Docker Hub. +pub(super) fn get_docker_download_token(client: &Client, image_name: &str) -> Result { + let token_json: serde_json::Value = client + .get(format!("https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/{image_name}:pull")) + .send().with_context(|| "Could not send request for anonymous authentication".to_string())? + .json().with_context(|| "Failed to parse JSON response for anonymous authentication".to_string())?; + + match token_json["token"].as_str().with_context(|| "Failed to get token from anon auth response".to_string()) { + Ok(t) => Ok(t.to_owned()), + Err(e) => Err(e) + } +} diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 3990f5c..6d22be7 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -6,12 +6,12 @@ use crate::cli_args::CliArgs; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use crate::image_builder::merge_layer; +use crate::loader::download::download_image_fs; mod cli_args; mod image_builder; -mod image_loader; mod initramfs_generator; -mod errors; +mod loader; fn run( args: CliArgs, @@ -22,7 +22,10 @@ fn run( let path = Path::new(output_subdir.as_path()); // image downloading and unpacking - let layers_paths = image_loader::download_image_fs(&args.image_name, layers_subdir)?; + let layers_paths = match download_image_fs(&args.image_name, layers_subdir) { + Err(e) => bail!(e), + Ok(e) => e + }; debug!("Layers' paths: {:?}", layers_paths); // reconstructing image with overlayfs @@ -61,10 +64,10 @@ fn main() -> Result<()> { ); if let Err(e) = run( - args, - args.temp_directory.clone().join("layers/"), - args.temp_directory.clone().join("output/"), - args.temp_directory.clone().join("overlay/") + args.clone(), + args.clone().temp_directory.clone().join("layers/"), + args.clone().temp_directory.clone().join("output/"), + args.clone().temp_directory.clone().join("overlay/") ) { error!(error = ?e, "encountered error while running"); Err(e) From f5eaa4d4e8f7c3d4cfed664de0becbc9facab867 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 10:05:38 +0200 Subject: [PATCH 16/25] refactor: improve horizontality of code in loader submodules Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/loader/download.rs | 145 +++++++++++++++--------------- src/fs-gen/src/loader/utils.rs | 4 +- src/fs-gen/src/main.rs | 2 +- 3 files changed, 76 insertions(+), 75 deletions(-) diff --git a/src/fs-gen/src/loader/download.rs b/src/fs-gen/src/loader/download.rs index f2f6e44..226478a 100644 --- a/src/fs-gen/src/loader/download.rs +++ b/src/fs-gen/src/loader/download.rs @@ -1,10 +1,9 @@ -use flate2::read::GzDecoder; -use reqwest::blocking::{Client, Response}; -use std::fs::create_dir; +use reqwest::blocking::Client; +use std::fs::create_dir_all; use std::path::PathBuf; -use tar::Archive; use tracing::{debug, info}; use anyhow::{Context, Result}; +use serde_json::Value; use crate::loader::errors::ImageLoaderError; use crate::loader::utils::{get_docker_download_token, unpack_tarball}; @@ -12,84 +11,90 @@ pub(crate) fn download_image_fs( image_name: &str, output_file: PathBuf, ) -> Result, ImageLoaderError> { + info!("Downloading image..."); + // Get image's name and tag let image_and_tag: Vec<&str> = image_name.split(':').collect(); - - let tag = if image_and_tag.len() < 2 { - "latest" - } else { - image_and_tag[1] - }; let image_name = image_and_tag[0]; + let tag = if image_and_tag.len() < 2 { "latest" } else { image_and_tag[1] }; - // Get download token + // Get download token and download manifest let client = Client::new(); let token = &get_docker_download_token(&client, image_name)?; - - // Download image manifest - let mut manifest_json = download_manifest(&client, token, image_name, tag) + let manifest = download_manifest(&client, token, image_name, tag) .map_err(|e| ImageLoaderError::Error { source: e })?; - // Verify if it's a manifest or a manifest list - let mut layers = manifest_json["layers"].as_array(); - - if layers.is_none() { - let manifests = manifest_json["manifests"].as_array(); - match manifests { - None => Err(ImageLoaderError::ManifestNotFound(image_name.to_string(), tag.to_string()))?, - Some(m) => { - debug!("Manifest list found. Looking for an amd64 manifest..."); - // TODO: implement other than amd64? - // Get a manifest for amd64 architecture from the manifest list - let amd64_manifest = m.iter().find(|manifest| { - manifest["platform"].as_object().unwrap()["architecture"] - .as_str() - .unwrap() - == "amd64" - }); - - match amd64_manifest { - None => Err(ImageLoaderError::UnsupportedArchitecture("amd64".to_string()))?, - Some(m) => { - info!("Downloading image..."); - debug!("Downloading manifest for amd64 architecture..."); - - manifest_json = download_manifest( - &client, - token, - image_name, - m["digest"].as_str().unwrap() - ).map_err(|e| ImageLoaderError::Error { source: e })?; - - layers = manifest_json["layers"].as_array(); - if layers.is_none() { - Err(ImageLoaderError::LayersNotFound)? - } - } - } - } - } + + if let Some(layers) = manifest["layers"].as_array() { + // We have layers already, no need to look into sub-manifests. + info!("Found layers in manifest"); + create_dir_all(&output_file).with_context(|| "Could not create output directory for image downloading")?; + return download_layers( + layers, + &client, + token, + image_name, + &output_file + ).map_err(|e| ImageLoaderError::Error { source: e }) } - let _ = create_dir(&output_file); + // Below, we assume there are no layers found. + // We dig into sub-manifests to try and find a suitable one to download, with hopefully layers inside. - download_layers( - layers.unwrap(), - &client, - token, - image_name, - &output_file - ).map_err(|e| ImageLoaderError::Error { source: e }) + let manifest_list = match manifest["manifests"].as_array() { + // No sub-manifests found, we throw an error. + None => Err(ImageLoaderError::ManifestNotFound(image_name.to_string(), tag.to_string()))?, + Some(m) => m + }; + info!(architecture = "amd64", "Manifest list found. Looking for an architecture-specific manifest..."); + + // TODO: implement other than amd64? + let amd64_submanifest = manifest_list.iter().find(|manifest| { + manifest["platform"].as_object().unwrap()["architecture"] + .as_str() + .unwrap() + == "amd64" + }); + + let submanifest = match amd64_submanifest { + None => Err(ImageLoaderError::UnsupportedArchitecture("amd64".to_string()))?, + Some(m) => { + debug!("Downloading architecture-specific manifest"); + + let submanifest = download_manifest( + &client, + token, + image_name, + m["digest"].as_str().unwrap() + ).map_err(|e| ImageLoaderError::Error { source: e })?; + + submanifest + } + }; + + match submanifest["layers"].as_array() { + None => Err(ImageLoaderError::LayersNotFound)?, + Some(layers) => { + create_dir_all(&output_file).with_context(|| "Could not create output directory for image downloading")?; + return download_layers( + layers, + &client, + token, + image_name, + &output_file + ).map_err(|e| ImageLoaderError::Error { source: e }) + } + } } -fn download_manifest(client: &Client, token: &str, image_name: &str, digest: &str) -> Result { +fn download_manifest(client: &Client, token: &str, image_name: &str, digest: &str) -> Result { // Query Docker Hub API to get the image manifest let manifest_url = format!( "https://registry-1.docker.io/v2/library/{}/manifests/{}", image_name, digest ); - let manifest: serde_json::Value = client + let manifest: Value = client .get(manifest_url) .header( "Accept", @@ -113,15 +118,15 @@ fn download_manifest(client: &Client, token: &str, image_name: &str, digest: &st } fn download_layers( - layers: &Vec, + layers: &Vec, client: &Client, token: &str, image_name: &str, output_dir: &PathBuf, ) -> Result> { - let mut layer_paths = Vec::new(); + info!("Downloading and unpacking layers..."); - debug!("Downloading and unpacking layers..."); + let mut layer_paths = Vec::new(); // Download and unpack each layer for layer in layers { @@ -137,13 +142,9 @@ fn download_layers( debug!("starting to decode layer with digest '{}'", digest); - let tar = GzDecoder::new(response); - - let mut output_path = PathBuf::new(); - output_path.push(output_dir); - output_path.push(digest); + let output_path = output_dir.join(digest); - unpack_tarball(tar, &output_path)?; + unpack_tarball(response, &output_path)?; debug!("layer '{}' unpacked", digest); layer_paths.push(output_path); } diff --git a/src/fs-gen/src/loader/utils.rs b/src/fs-gen/src/loader/utils.rs index 98ab949..6f49775 100644 --- a/src/fs-gen/src/loader/utils.rs +++ b/src/fs-gen/src/loader/utils.rs @@ -5,8 +5,8 @@ use tar::Archive; use anyhow::{Context, Result}; /// Unpack the tarball to a given directory. -pub(super) fn unpack_tarball(tar: GzDecoder, output_dir: &PathBuf) -> Result<()> { - Archive::new(tar).unpack(output_dir.clone()) +pub(super) fn unpack_tarball(response: Response, output_dir: &PathBuf) -> Result<()> { + Archive::new(GzDecoder::new(response)).unpack(output_dir.clone()) .with_context(|| format!("Failed to unpack tarball to {}", output_dir.display()))?; Ok(()) } diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 6d22be7..bffd265 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,7 +1,7 @@ use std::{fs::remove_dir_all, path::Path}; use std::path::PathBuf; use tracing::{debug, error, info, Level}; -use anyhow::{Result, Error, bail, Context}; +use anyhow::{Result, bail, Context}; use crate::cli_args::CliArgs; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; From bb5a6f6522d2783bd8e2e94f0ea2b272e024d8d1 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:38:31 +0200 Subject: [PATCH 17/25] lint: correct for clippy Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/loader/download.rs | 14 +++++++------- src/fs-gen/src/loader/mod.rs | 2 +- src/fs-gen/src/loader/utils.rs | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/fs-gen/src/loader/download.rs b/src/fs-gen/src/loader/download.rs index 226478a..31f9b47 100644 --- a/src/fs-gen/src/loader/download.rs +++ b/src/fs-gen/src/loader/download.rs @@ -1,6 +1,6 @@ use reqwest::blocking::Client; use std::fs::create_dir_all; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tracing::{debug, info}; use anyhow::{Context, Result}; use serde_json::Value; @@ -61,14 +61,14 @@ pub(crate) fn download_image_fs( Some(m) => { debug!("Downloading architecture-specific manifest"); - let submanifest = download_manifest( + + + download_manifest( &client, token, image_name, m["digest"].as_str().unwrap() - ).map_err(|e| ImageLoaderError::Error { source: e })?; - - submanifest + ).map_err(|e| ImageLoaderError::Error { source: e })? } }; @@ -76,7 +76,7 @@ pub(crate) fn download_image_fs( None => Err(ImageLoaderError::LayersNotFound)?, Some(layers) => { create_dir_all(&output_file).with_context(|| "Could not create output directory for image downloading")?; - return download_layers( + download_layers( layers, &client, token, @@ -122,7 +122,7 @@ fn download_layers( client: &Client, token: &str, image_name: &str, - output_dir: &PathBuf, + output_dir: &Path, ) -> Result> { info!("Downloading and unpacking layers..."); diff --git a/src/fs-gen/src/loader/mod.rs b/src/fs-gen/src/loader/mod.rs index 866053d..8f2745d 100644 --- a/src/fs-gen/src/loader/mod.rs +++ b/src/fs-gen/src/loader/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod download; pub(crate) mod errors; -pub(self) mod utils; + mod utils; diff --git a/src/fs-gen/src/loader/utils.rs b/src/fs-gen/src/loader/utils.rs index 6f49775..64afb73 100644 --- a/src/fs-gen/src/loader/utils.rs +++ b/src/fs-gen/src/loader/utils.rs @@ -1,12 +1,12 @@ use flate2::read::GzDecoder; use reqwest::blocking::{Client, Response}; -use std::path::PathBuf; +use std::path::{Path}; use tar::Archive; use anyhow::{Context, Result}; /// Unpack the tarball to a given directory. -pub(super) fn unpack_tarball(response: Response, output_dir: &PathBuf) -> Result<()> { - Archive::new(GzDecoder::new(response)).unpack(output_dir.clone()) +pub(super) fn unpack_tarball(response: Response, output_dir: &Path) -> Result<()> { + Archive::new(GzDecoder::new(response)).unpack(output_dir) .with_context(|| format!("Failed to unpack tarball to {}", output_dir.display()))?; Ok(()) } From 2dd018d9a4ac404dcdf79366c9d0b112dcf3c244 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:47:47 +0200 Subject: [PATCH 18/25] feat: implement architecture choice when building initramfs Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/cli_args.rs | 3 +++ src/fs-gen/src/loader/download.rs | 14 ++++++-------- src/fs-gen/src/main.rs | 3 ++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/fs-gen/src/cli_args.rs b/src/fs-gen/src/cli_args.rs index 011406a..718f80c 100644 --- a/src/fs-gen/src/cli_args.rs +++ b/src/fs-gen/src/cli_args.rs @@ -31,6 +31,9 @@ pub struct CliArgs { #[arg(short='i', long="init", default_value=None)] pub initfile_path: Option, + #[arg(long="arch", default_value="amd64")] + pub architecture: String, + #[arg(short='d', long="debug", action=ArgAction::SetTrue)] pub debug: bool, } diff --git a/src/fs-gen/src/loader/download.rs b/src/fs-gen/src/loader/download.rs index 31f9b47..8c85d0b 100644 --- a/src/fs-gen/src/loader/download.rs +++ b/src/fs-gen/src/loader/download.rs @@ -9,6 +9,7 @@ use crate::loader::utils::{get_docker_download_token, unpack_tarball}; pub(crate) fn download_image_fs( image_name: &str, + architecture: &str, output_file: PathBuf, ) -> Result, ImageLoaderError> { info!("Downloading image..."); @@ -46,23 +47,20 @@ pub(crate) fn download_image_fs( None => Err(ImageLoaderError::ManifestNotFound(image_name.to_string(), tag.to_string()))?, Some(m) => m }; - info!(architecture = "amd64", "Manifest list found. Looking for an architecture-specific manifest..."); + info!(architecture, "Manifest list found. Looking for an architecture-specific manifest..."); - // TODO: implement other than amd64? - let amd64_submanifest = manifest_list.iter().find(|manifest| { + let arch_specific_manifest = manifest_list.iter().find(|manifest| { manifest["platform"].as_object().unwrap()["architecture"] .as_str() .unwrap() - == "amd64" + == architecture }); - let submanifest = match amd64_submanifest { - None => Err(ImageLoaderError::UnsupportedArchitecture("amd64".to_string()))?, + let submanifest = match arch_specific_manifest { + None => Err(ImageLoaderError::UnsupportedArchitecture(architecture.to_string()))?, Some(m) => { debug!("Downloading architecture-specific manifest"); - - download_manifest( &client, token, diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index bffd265..9a9b28a 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -22,7 +22,7 @@ fn run( let path = Path::new(output_subdir.as_path()); // image downloading and unpacking - let layers_paths = match download_image_fs(&args.image_name, layers_subdir) { + let layers_paths = match download_image_fs(&args.image_name, &args.architecture, layers_subdir) { Err(e) => bail!(e), Ok(e) => e }; @@ -59,6 +59,7 @@ fn main() -> Result<()> { output_file = ?args.output_file, temp_dir = ?args.temp_directory, initfile_path = ?args.initfile_path, + architecture = args.architecture, debug = args.debug, "arguments:", ); From 971d355e602ada672412f7ad5f42ca2d9b271ca6 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:32:21 +0200 Subject: [PATCH 19/25] refactor: get subdirs in run function Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/main.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 9a9b28a..4cc2e1d 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,5 +1,4 @@ use std::{fs::remove_dir_all, path::Path}; -use std::path::PathBuf; use tracing::{debug, error, info, Level}; use anyhow::{Result, bail, Context}; use crate::cli_args::CliArgs; @@ -15,10 +14,11 @@ mod loader; fn run( args: CliArgs, - layers_subdir: PathBuf, - output_subdir: PathBuf, - overlay_subdir: PathBuf, ) -> Result<()> { + let layers_subdir = args.temp_directory.join("layers/"); + let output_subdir = args.temp_directory.join("output/"); + let overlay_subdir = args.temp_directory.join("overlay/"); + let path = Path::new(output_subdir.as_path()); // image downloading and unpacking @@ -64,12 +64,7 @@ fn main() -> Result<()> { "arguments:", ); - if let Err(e) = run( - args.clone(), - args.clone().temp_directory.clone().join("layers/"), - args.clone().temp_directory.clone().join("output/"), - args.clone().temp_directory.clone().join("overlay/") - ) { + if let Err(e) = run(args) { error!(error = ?e, "encountered error while running"); Err(e) } else { From ccaaf20b2fefa3cec295484a78850705f7e9ce81 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:54:57 +0200 Subject: [PATCH 20/25] feat: add warning when no architecture in manifest Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/loader/download.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/fs-gen/src/loader/download.rs b/src/fs-gen/src/loader/download.rs index 8c85d0b..bfa3e38 100644 --- a/src/fs-gen/src/loader/download.rs +++ b/src/fs-gen/src/loader/download.rs @@ -1,7 +1,7 @@ use reqwest::blocking::Client; use std::fs::create_dir_all; use std::path::{Path, PathBuf}; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use anyhow::{Context, Result}; use serde_json::Value; use crate::loader::errors::ImageLoaderError; @@ -29,6 +29,10 @@ pub(crate) fn download_image_fs( if let Some(layers) = manifest["layers"].as_array() { // We have layers already, no need to look into sub-manifests. info!("Found layers in manifest"); + warn!( + architecture, + "Manifest did not specify architecture, the initramfs may not work for the requested architecture" + ); create_dir_all(&output_file).with_context(|| "Could not create output directory for image downloading")?; return download_layers( layers, From 2b38a1a2a9c042cf183310ee866a0cc71bbe07e6 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 17:52:03 +0200 Subject: [PATCH 21/25] feat: add new initfile version Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/resources/initfile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/fs-gen/resources/initfile b/src/fs-gen/resources/initfile index 72144fb..b405af9 100644 --- a/src/fs-gen/resources/initfile +++ b/src/fs-gen/resources/initfile @@ -1,11 +1,21 @@ -#!/bin/sh +#! /bin/sh # # Cloudlet initramfs generation # mount -t devtmpfs dev /dev mount -t proc proc /proc mount -t sysfs sysfs /sys + ip link set up dev lo -exec /sbin/getty -n -l /bin/sh 115200 /dev/console -poweroff -f +slattach -L /dev/ttyS1& + +while ! ifconfig sl0 &> /dev/null; do + sleep 1 +done + +ifconfig sl0 172.30.0.11 netmask 255.255.0.0 up + +/agent + +reboot From 88f59965570952fbc50baa7476d8ccc99173bb65 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Thu, 25 Apr 2024 18:05:59 +0200 Subject: [PATCH 22/25] chore: remove empty test file Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/test | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/fs-gen/test diff --git a/src/fs-gen/test b/src/fs-gen/test deleted file mode 100644 index e69de29..0000000 From 69093ed225bc5a8f8cd4582d966ca0b79e5da5eb Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:42:47 +0200 Subject: [PATCH 23/25] refactor: suppress info level for fuse backend crate Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/Cargo.toml | 2 +- src/fs-gen/src/main.rs | 34 +++++++++++++++++++++++----------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/fs-gen/Cargo.toml b/src/fs-gen/Cargo.toml index a131d9e..eb0b4eb 100644 --- a/src/fs-gen/Cargo.toml +++ b/src/fs-gen/Cargo.toml @@ -20,5 +20,5 @@ tar = "0.4.40" validator = { version = "0.17.0", features = ["derive"] } anyhow = "1.0.82" tracing = "0.1.40" -tracing-subscriber = "0.3.18" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } thiserror = "1.0.59" diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 4cc2e1d..07ede5f 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,8 +1,10 @@ use std::{fs::remove_dir_all, path::Path}; -use tracing::{debug, error, info, Level}; +use tracing::{debug, error, info}; use anyhow::{Result, bail, Context}; -use crate::cli_args::CliArgs; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::filter::EnvFilter; +use crate::cli_args::CliArgs; use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use crate::image_builder::merge_layer; use crate::loader::download::download_image_fs; @@ -16,10 +18,9 @@ fn run( args: CliArgs, ) -> Result<()> { let layers_subdir = args.temp_directory.join("layers/"); - let output_subdir = args.temp_directory.join("output/"); let overlay_subdir = args.temp_directory.join("overlay/"); - - let path = Path::new(output_subdir.as_path()); + let _binding = args.temp_directory.join("output/"); + let output_subdir = _binding.as_path(); // image downloading and unpacking let layers_paths = match download_image_fs(&args.image_name, &args.architecture, layers_subdir) { @@ -29,12 +30,12 @@ fn run( debug!("Layers' paths: {:?}", layers_paths); // reconstructing image with overlayfs - merge_layer(&layers_paths, path, &overlay_subdir)?; + merge_layer(&layers_paths, output_subdir, &overlay_subdir)?; // building initramfs - create_init_file(path, args.initfile_path)?; - insert_agent(path, args.agent_host_path)?; - generate_initramfs(path, Path::new(args.output_file.as_path()))?; + create_init_file(output_subdir, args.initfile_path)?; + insert_agent(output_subdir, args.agent_host_path)?; + generate_initramfs(output_subdir, Path::new(args.output_file.as_path()))?; // cleanup of temporary directory remove_dir_all(args.temp_directory.clone()) @@ -47,10 +48,21 @@ fn main() -> Result<()> { let args = CliArgs::get_args(); tracing_subscriber::fmt() - .with_max_level(if args.debug { Level::DEBUG } else { Level::INFO }) + .with_env_filter( + EnvFilter::builder() + .with_default_directive( + (if args.debug { LevelFilter::DEBUG } else { LevelFilter::INFO }).into() + ) + .from_env()? + .add_directive("fuse_backend_rs=warn".parse()?) + ) .init(); + + // tracing_subscriber::fmt() + // .with_max_level(if args.debug { Level::DEBUG } else { Level::INFO }) + // .init(); - info!("Cloudlet initramfs generator v{}", env!("CARGO_PKG_VERSION")); + info!("Cloudlet initramfs generator: '{}' v{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); info!("Generating for image '{}'", args.image_name); debug!( From a36237788bd172e97719f09962b0f6615545aee7 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Fri, 26 Apr 2024 12:48:29 +0200 Subject: [PATCH 24/25] feat: add info logging for layers merging Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/image_builder.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/fs-gen/src/image_builder.rs b/src/fs-gen/src/image_builder.rs index 026b990..b34221a 100644 --- a/src/fs-gen/src/image_builder.rs +++ b/src/fs-gen/src/image_builder.rs @@ -13,7 +13,7 @@ use fuse_backend_rs::{ passthrough::{self, PassthroughFs}, transport::{FuseChannel, FuseSession}, }; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; static FILE_EXISTS_ERROR: i32 = 17; @@ -72,6 +72,8 @@ fn ensure_folder_created(output_folder: &Path) -> Result<()> { /// merge_layer(vec!["source/layer_1", "source/layer_2"], "/tmp/fused_layers", "/tmp") /// ``` pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Path) -> Result<()> { + info!("Starting to merge layers..."); + // Stack all lower layers let mut lower_layers = Vec::new(); for lower in blob_paths { @@ -134,6 +136,8 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Pa .with_context(|| "Failed to unmount the fuse session".to_string())?; let _ = handle.join(); + + info!("Finished merging layers!"); Ok(()) } From 360ce44acc6a307ae43b3652f690957fe0133067 Mon Sep 17 00:00:00 2001 From: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> Date: Fri, 26 Apr 2024 13:10:21 +0200 Subject: [PATCH 25/25] chore: cargo fmt Signed-off-by: BioTheWolff <47079795+BioTheWolff@users.noreply.github.com> --- src/fs-gen/src/cli_args.rs | 4 +- src/fs-gen/src/image_builder.rs | 2 +- src/fs-gen/src/initramfs_generator.rs | 16 ++--- src/fs-gen/src/loader/download.rs | 88 +++++++++++++++------------ src/fs-gen/src/loader/errors.rs | 4 +- src/fs-gen/src/loader/mod.rs | 2 +- src/fs-gen/src/loader/utils.rs | 14 +++-- src/fs-gen/src/main.rs | 32 ++++++---- 8 files changed, 91 insertions(+), 71 deletions(-) diff --git a/src/fs-gen/src/cli_args.rs b/src/fs-gen/src/cli_args.rs index 718f80c..a736113 100644 --- a/src/fs-gen/src/cli_args.rs +++ b/src/fs-gen/src/cli_args.rs @@ -1,6 +1,6 @@ use std::{env, path::PathBuf}; -use clap::{command, error::ErrorKind, CommandFactory, Parser, ArgAction}; +use clap::{command, error::ErrorKind, ArgAction, CommandFactory, Parser}; use regex::Regex; use once_cell::sync::Lazy; @@ -31,7 +31,7 @@ pub struct CliArgs { #[arg(short='i', long="init", default_value=None)] pub initfile_path: Option, - #[arg(long="arch", default_value="amd64")] + #[arg(long = "arch", default_value = "amd64")] pub architecture: String, #[arg(short='d', long="debug", action=ArgAction::SetTrue)] diff --git a/src/fs-gen/src/image_builder.rs b/src/fs-gen/src/image_builder.rs index b34221a..fa8803a 100644 --- a/src/fs-gen/src/image_builder.rs +++ b/src/fs-gen/src/image_builder.rs @@ -136,7 +136,7 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Pa .with_context(|| "Failed to unmount the fuse session".to_string())?; let _ = handle.join(); - + info!("Finished merging layers!"); Ok(()) } diff --git a/src/fs-gen/src/initramfs_generator.rs b/src/fs-gen/src/initramfs_generator.rs index f9c5d40..c216fa0 100644 --- a/src/fs-gen/src/initramfs_generator.rs +++ b/src/fs-gen/src/initramfs_generator.rs @@ -1,10 +1,10 @@ -use std::fs::{File, Permissions, copy as fscopy}; -use std::io::{Write, copy as iocopy}; +use anyhow::{Context, Result}; +use std::fs::{copy as fscopy, File, Permissions}; +use std::io::{copy as iocopy, Write}; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use tracing::info; -use anyhow::{Context, Result}; const INIT_FILE: &str = include_str!("../resources/initfile"); @@ -39,8 +39,8 @@ pub fn insert_agent(destination: &Path, agent_path: PathBuf) -> Result<()> { file.set_permissions(Permissions::from_mode(0o755)) .with_context(|| "Failed to set permissions for agent file".to_string())?; - let mut agent = File::open(agent_path) - .with_context(|| "Could not open host agent file".to_string())?; + let mut agent = + File::open(agent_path).with_context(|| "Could not open host agent file".to_string())?; iocopy(&mut agent, &mut file) .with_context(|| "Failed to copy agent contents from host to destination".to_string())?; @@ -65,9 +65,9 @@ pub fn generate_initramfs(root_directory: &Path, output: &Path) -> Result<()> { .spawn() .with_context(|| "Failed to package initramfs into bundle".to_string())?; - command - .wait() - .with_context(|| "Encountered exception while waiting for bundling to finish".to_string())?; + command.wait().with_context(|| { + "Encountered exception while waiting for bundling to finish".to_string() + })?; info!("Initramfs generated!"); diff --git a/src/fs-gen/src/loader/download.rs b/src/fs-gen/src/loader/download.rs index bfa3e38..1cc8c65 100644 --- a/src/fs-gen/src/loader/download.rs +++ b/src/fs-gen/src/loader/download.rs @@ -1,11 +1,11 @@ +use crate::loader::errors::ImageLoaderError; +use crate::loader::utils::{get_docker_download_token, unpack_tarball}; +use anyhow::{Context, Result}; use reqwest::blocking::Client; +use serde_json::Value; use std::fs::create_dir_all; use std::path::{Path, PathBuf}; use tracing::{debug, info, warn}; -use anyhow::{Context, Result}; -use serde_json::Value; -use crate::loader::errors::ImageLoaderError; -use crate::loader::utils::{get_docker_download_token, unpack_tarball}; pub(crate) fn download_image_fs( image_name: &str, @@ -17,7 +17,11 @@ pub(crate) fn download_image_fs( // Get image's name and tag let image_and_tag: Vec<&str> = image_name.split(':').collect(); let image_name = image_and_tag[0]; - let tag = if image_and_tag.len() < 2 { "latest" } else { image_and_tag[1] }; + let tag = if image_and_tag.len() < 2 { + "latest" + } else { + image_and_tag[1] + }; // Get download token and download manifest let client = Client::new(); @@ -25,7 +29,6 @@ pub(crate) fn download_image_fs( let manifest = download_manifest(&client, token, image_name, tag) .map_err(|e| ImageLoaderError::Error { source: e })?; - if let Some(layers) = manifest["layers"].as_array() { // We have layers already, no need to look into sub-manifests. info!("Found layers in manifest"); @@ -33,14 +36,10 @@ pub(crate) fn download_image_fs( architecture, "Manifest did not specify architecture, the initramfs may not work for the requested architecture" ); - create_dir_all(&output_file).with_context(|| "Could not create output directory for image downloading")?; - return download_layers( - layers, - &client, - token, - image_name, - &output_file - ).map_err(|e| ImageLoaderError::Error { source: e }) + create_dir_all(&output_file) + .with_context(|| "Could not create output directory for image downloading")?; + return download_layers(layers, &client, token, image_name, &output_file) + .map_err(|e| ImageLoaderError::Error { source: e }); } // Below, we assume there are no layers found. @@ -48,10 +47,16 @@ pub(crate) fn download_image_fs( let manifest_list = match manifest["manifests"].as_array() { // No sub-manifests found, we throw an error. - None => Err(ImageLoaderError::ManifestNotFound(image_name.to_string(), tag.to_string()))?, - Some(m) => m + None => Err(ImageLoaderError::ManifestNotFound( + image_name.to_string(), + tag.to_string(), + ))?, + Some(m) => m, }; - info!(architecture, "Manifest list found. Looking for an architecture-specific manifest..."); + info!( + architecture, + "Manifest list found. Looking for an architecture-specific manifest..." + ); let arch_specific_manifest = manifest_list.iter().find(|manifest| { manifest["platform"].as_object().unwrap()["architecture"] @@ -61,35 +66,34 @@ pub(crate) fn download_image_fs( }); let submanifest = match arch_specific_manifest { - None => Err(ImageLoaderError::UnsupportedArchitecture(architecture.to_string()))?, + None => Err(ImageLoaderError::UnsupportedArchitecture( + architecture.to_string(), + ))?, Some(m) => { debug!("Downloading architecture-specific manifest"); - download_manifest( - &client, - token, - image_name, - m["digest"].as_str().unwrap() - ).map_err(|e| ImageLoaderError::Error { source: e })? + download_manifest(&client, token, image_name, m["digest"].as_str().unwrap()) + .map_err(|e| ImageLoaderError::Error { source: e })? } }; - + match submanifest["layers"].as_array() { None => Err(ImageLoaderError::LayersNotFound)?, Some(layers) => { - create_dir_all(&output_file).with_context(|| "Could not create output directory for image downloading")?; - download_layers( - layers, - &client, - token, - image_name, - &output_file - ).map_err(|e| ImageLoaderError::Error { source: e }) + create_dir_all(&output_file) + .with_context(|| "Could not create output directory for image downloading")?; + download_layers(layers, &client, token, image_name, &output_file) + .map_err(|e| ImageLoaderError::Error { source: e }) } } } -fn download_manifest(client: &Client, token: &str, image_name: &str, digest: &str) -> Result { +fn download_manifest( + client: &Client, + token: &str, + image_name: &str, + digest: &str, +) -> Result { // Query Docker Hub API to get the image manifest let manifest_url = format!( "https://registry-1.docker.io/v2/library/{}/manifests/{}", @@ -108,8 +112,10 @@ fn download_manifest(client: &Client, token: &str, image_name: &str, digest: &st ) .header("Accept", "application/vnd.oci.image.manifest.v1+json") .bearer_auth(token) - .send().with_context(|| "Could not send request to get manifest data".to_string())? - .json().with_context(|| "Failed to parse manifest to JSON".to_string())?; + .send() + .with_context(|| "Could not send request to get manifest data".to_string())? + .json() + .with_context(|| "Failed to parse manifest to JSON".to_string())?; debug!( manifest = ?manifest, @@ -132,15 +138,19 @@ fn download_layers( // Download and unpack each layer for layer in layers { - let digest = layer["digest"].as_str() + let digest = layer["digest"] + .as_str() .with_context(|| "Failed to get digest for layer".to_string())?; let layer_url = format!( "https://registry-1.docker.io/v2/library/{}/blobs/{}", image_name, digest ); - let response = client.get(&layer_url).bearer_auth(token) - .send().with_context(|| format!("Could not send request for layer digest '{digest}'"))?; + let response = client + .get(&layer_url) + .bearer_auth(token) + .send() + .with_context(|| format!("Could not send request for layer digest '{digest}'"))?; debug!("starting to decode layer with digest '{}'", digest); diff --git a/src/fs-gen/src/loader/errors.rs b/src/fs-gen/src/loader/errors.rs index b6fda91..d262c04 100644 --- a/src/fs-gen/src/loader/errors.rs +++ b/src/fs-gen/src/loader/errors.rs @@ -17,9 +17,7 @@ pub(crate) enum ImageLoaderError { /// Encountered an error during the flow. #[error("Image loading error: {}", .source)] - Error { - source: anyhow::Error - } + Error { source: anyhow::Error }, } impl From for ImageLoaderError { diff --git a/src/fs-gen/src/loader/mod.rs b/src/fs-gen/src/loader/mod.rs index 8f2745d..f79903f 100644 --- a/src/fs-gen/src/loader/mod.rs +++ b/src/fs-gen/src/loader/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod download; pub(crate) mod errors; - mod utils; +mod utils; diff --git a/src/fs-gen/src/loader/utils.rs b/src/fs-gen/src/loader/utils.rs index 64afb73..d65d247 100644 --- a/src/fs-gen/src/loader/utils.rs +++ b/src/fs-gen/src/loader/utils.rs @@ -1,12 +1,13 @@ +use anyhow::{Context, Result}; use flate2::read::GzDecoder; use reqwest::blocking::{Client, Response}; -use std::path::{Path}; +use std::path::Path; use tar::Archive; -use anyhow::{Context, Result}; /// Unpack the tarball to a given directory. pub(super) fn unpack_tarball(response: Response, output_dir: &Path) -> Result<()> { - Archive::new(GzDecoder::new(response)).unpack(output_dir) + Archive::new(GzDecoder::new(response)) + .unpack(output_dir) .with_context(|| format!("Failed to unpack tarball to {}", output_dir.display()))?; Ok(()) } @@ -18,8 +19,11 @@ pub(super) fn get_docker_download_token(client: &Client, image_name: &str) -> Re .send().with_context(|| "Could not send request for anonymous authentication".to_string())? .json().with_context(|| "Failed to parse JSON response for anonymous authentication".to_string())?; - match token_json["token"].as_str().with_context(|| "Failed to get token from anon auth response".to_string()) { + match token_json["token"] + .as_str() + .with_context(|| "Failed to get token from anon auth response".to_string()) + { Ok(t) => Ok(t.to_owned()), - Err(e) => Err(e) + Err(e) => Err(e), } } diff --git a/src/fs-gen/src/main.rs b/src/fs-gen/src/main.rs index 07ede5f..045e1c0 100644 --- a/src/fs-gen/src/main.rs +++ b/src/fs-gen/src/main.rs @@ -1,12 +1,12 @@ +use anyhow::{bail, Context, Result}; use std::{fs::remove_dir_all, path::Path}; -use tracing::{debug, error, info}; -use anyhow::{Result, bail, Context}; use tracing::level_filters::LevelFilter; +use tracing::{debug, error, info}; use tracing_subscriber::filter::EnvFilter; use crate::cli_args::CliArgs; -use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use crate::image_builder::merge_layer; +use crate::initramfs_generator::{create_init_file, generate_initramfs, insert_agent}; use crate::loader::download::download_image_fs; mod cli_args; @@ -14,18 +14,17 @@ mod image_builder; mod initramfs_generator; mod loader; -fn run( - args: CliArgs, -) -> Result<()> { +fn run(args: CliArgs) -> Result<()> { let layers_subdir = args.temp_directory.join("layers/"); let overlay_subdir = args.temp_directory.join("overlay/"); let _binding = args.temp_directory.join("output/"); let output_subdir = _binding.as_path(); // image downloading and unpacking - let layers_paths = match download_image_fs(&args.image_name, &args.architecture, layers_subdir) { + let layers_paths = match download_image_fs(&args.image_name, &args.architecture, layers_subdir) + { Err(e) => bail!(e), - Ok(e) => e + Ok(e) => e, }; debug!("Layers' paths: {:?}", layers_paths); @@ -51,18 +50,27 @@ fn main() -> Result<()> { .with_env_filter( EnvFilter::builder() .with_default_directive( - (if args.debug { LevelFilter::DEBUG } else { LevelFilter::INFO }).into() + (if args.debug { + LevelFilter::DEBUG + } else { + LevelFilter::INFO + }) + .into(), ) .from_env()? - .add_directive("fuse_backend_rs=warn".parse()?) + .add_directive("fuse_backend_rs=warn".parse()?), ) .init(); - + // tracing_subscriber::fmt() // .with_max_level(if args.debug { Level::DEBUG } else { Level::INFO }) // .init(); - info!("Cloudlet initramfs generator: '{}' v{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); + info!( + "Cloudlet initramfs generator: '{}' v{}", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_VERSION") + ); info!("Generating for image '{}'", args.image_name); debug!(