Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(fs-gen): stable version with agent insertion #30

Merged
merged 25 commits into from
Apr 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
bcf927f
fix(image-builder): use the proper tmp path
Mathias-Boulay Apr 23, 2024
0b580e5
refactor: use a subfolder by default
Mathias-Boulay Apr 23, 2024
320bdfb
feat(arg-parsing): make the tag optional (#4)
Mathias-Boulay Apr 23, 2024
e2c3e26
refactor: change default temp directory
BioTheWolff Apr 23, 2024
526db41
feat: insert initfile from command-line argument
BioTheWolff Apr 23, 2024
eb4e670
refactor: load initfile using include_str
BioTheWolff Apr 23, 2024
0b08f32
feat: insert agent when building initramfs
BioTheWolff Apr 23, 2024
d962d89
chore: add tracing crate
BioTheWolff Apr 23, 2024
88412a1
feat: implement tracing for crate
BioTheWolff Apr 23, 2024
9d943d3
lint: remove unused import
BioTheWolff Apr 24, 2024
5a1dc24
refactor: implement better error handling for initramfs generation
BioTheWolff Apr 24, 2024
60c5b04
refactor: add error handling for image loading
BioTheWolff Apr 24, 2024
b0e4143
refactor: log errors for each soft unwrap
BioTheWolff Apr 24, 2024
1c1ac09
refactor: simplify error handling in main function
BioTheWolff Apr 24, 2024
e6bf37e
refactor: rework architecture for downloading and unpacking
BioTheWolff Apr 25, 2024
f5eaa4d
refactor: improve horizontality of code in loader submodules
BioTheWolff Apr 25, 2024
bb5a6f6
lint: correct for clippy
BioTheWolff Apr 25, 2024
2dd018d
feat: implement architecture choice when building initramfs
BioTheWolff Apr 25, 2024
971d355
refactor: get subdirs in run function
BioTheWolff Apr 25, 2024
ccaaf20
feat: add warning when no architecture in manifest
BioTheWolff Apr 25, 2024
2b38a1a
feat: add new initfile version
BioTheWolff Apr 25, 2024
88f5996
chore: remove empty test file
BioTheWolff Apr 25, 2024
69093ed
refactor: suppress info level for fuse backend crate
BioTheWolff Apr 26, 2024
a362377
feat: add info logging for layers merging
BioTheWolff Apr 26, 2024
360ce44
chore: cargo fmt
BioTheWolff Apr 26, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/fs-gen/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,6 @@ signal-hook = "0.3.17"
tar = "0.4.40"
validator = { version = "0.17.0", features = ["derive"] }
anyhow = "1.0.82"
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
thiserror = "1.0.59"
21 changes: 21 additions & 0 deletions src/fs-gen/resources/initfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#! /bin/sh
#
# Cloudlet initramfs generation
#
mount -t devtmpfs dev /dev
mount -t proc proc /proc
mount -t sysfs sysfs /sys

ip link set up dev lo

slattach -L /dev/ttyS1&

while ! ifconfig sl0 &> /dev/null; do
sleep 1
done

ifconfig sl0 172.30.0.11 netmask 255.255.0.0 up

/agent

reboot
26 changes: 17 additions & 9 deletions src/fs-gen/src/cli_args.rs
Original file line number Diff line number Diff line change
@@ -1,22 +1,25 @@
use std::{env, path::PathBuf};

use clap::{command, error::ErrorKind, CommandFactory, Parser};
use clap::{command, error::ErrorKind, ArgAction, CommandFactory, Parser};
use regex::Regex;

use once_cell::sync::Lazy;

// So, for any of you who may be scared, this is the regex from the OCI Distribution Sepcification for the image name + the tag
static RE_IMAGE_NAME: Lazy<Regex> = Lazy::new(|| {
Regex::new(r"[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(\/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*:[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}").unwrap()
Regex::new(r"[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*(/[a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*)*(?::[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127})?").unwrap()
});

/// Convert an OCI image into a CPIO file
#[derive(Parser, Debug)]
#[derive(Parser, Debug, Clone)]
#[command(version, about, long_about = None)]
pub struct CliArgs {
/// The name of the image to download
pub image_name: String,

/// The host path to the guest agent binary
pub agent_host_path: PathBuf,

/// The path to the output file
#[arg(short='o', long="output", default_value=get_default_output_file().into_os_string())]
pub output_file: PathBuf,
Expand All @@ -25,8 +28,14 @@ pub struct CliArgs {
#[arg(short='t', long="tempdir", default_value=get_default_temp_directory().into_os_string())]
pub temp_directory: PathBuf,

/// The host path to the guest agent binary
pub agent_host_path: PathBuf,
#[arg(short='i', long="init", default_value=None)]
pub initfile_path: Option<PathBuf>,

#[arg(long = "arch", default_value = "amd64")]
pub architecture: String,

#[arg(short='d', long="debug", action=ArgAction::SetTrue)]
pub debug: bool,
}

impl CliArgs {
Expand Down Expand Up @@ -66,13 +75,12 @@ impl CliArgs {
}
}

/// Get the default output path for the cpio file.
/// Get the default temporary directory for the current execution.
fn get_default_temp_directory() -> PathBuf {
let mut path = env::current_dir().unwrap();
path.push(".cloudlet_temp/");
path
PathBuf::from("/tmp/cloudlet-fs-gen")
}

/// Get the default output file path for the generated initramfs.
fn get_default_output_file() -> PathBuf {
let mut path = env::current_dir().unwrap();
path.push("initramfs.img");
Expand Down
42 changes: 26 additions & 16 deletions src/fs-gen/src/image_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use fuse_backend_rs::{
passthrough::{self, PassthroughFs},
transport::{FuseChannel, FuseSession},
};
use tracing::{debug, info, warn};

static FILE_EXISTS_ERROR: i32 = 17;

Expand Down Expand Up @@ -44,16 +45,21 @@ fn new_passthroughfs_layer(rootdir: &str) -> Result<BoxedLayer> {

/// Ensure a destination folder is created
fn ensure_folder_created(output_folder: &Path) -> Result<()> {
let result = fs::create_dir(output_folder);
let result = fs::create_dir_all(output_folder);

// If the file already exists, we're fine
if result.is_err()
&& result
.unwrap_err()
if result.is_err() {
let err = result.unwrap_err();

if err
.raw_os_error()
.is_some_and(|err_val| err_val != FILE_EXISTS_ERROR)
{
return Err(anyhow!("Failed to create folder"));
{
return Err(anyhow!(err)).context(format!(
"Failed to create folder: {}",
output_folder.to_string_lossy()
));
}
}

Ok(())
Expand All @@ -63,24 +69,27 @@ fn ensure_folder_created(output_folder: &Path) -> Result<()> {
/// It works by instantiating an overlay fs via FUSE then copying the files to the desired target
/// # Usage
/// ```
/// merge_layer(vec!["source/layer_1", "source/layer_2"], "/tmp/fused_layers")
/// merge_layer(vec!["source/layer_1", "source/layer_2"], "/tmp/fused_layers", "/tmp")
/// ```
pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path) -> Result<()> {
pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path, tmp_folder: &Path) -> Result<()> {
info!("Starting to merge layers...");

// Stack all lower layers
let mut lower_layers = Vec::new();
for lower in blob_paths {
lower_layers.push(Arc::new(new_passthroughfs_layer(&lower.to_string_lossy())?));
}

let mountpoint = Path::new("/tmp/cloudlet_internal");
let binding = tmp_folder.join("overlayfs_mountpoint");
let mountpoint = binding.as_path();
let fs_name = "cloudlet_overlay";

ensure_folder_created(mountpoint)?;
ensure_folder_created(output_folder)?;

// Setup the overlay fs config
let config = Config {
work: "/work".into(),
work: tmp_folder.join("work").to_string_lossy().into(),
mountpoint: output_folder.to_string_lossy().into(),
do_import: true,
..Default::default()
Expand Down Expand Up @@ -110,15 +119,15 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path) -> Result<()> {
let _ = server.svc_loop();
});

println!("copy starting !");
debug!("Starting copy...");
//So now we need to copy the files
dircpy::copy_dir(mountpoint, output_folder).with_context(|| {
format!(
"Failed to copy directories into the output folder: {}",
output_folder.to_string_lossy()
)
})?;
println!("copy finished");
debug!("Copy finished!");

// Unmount sessions so it can be re-used in later executions of the program
se.wake()
Expand All @@ -127,22 +136,23 @@ pub fn merge_layer(blob_paths: &[PathBuf], output_folder: &Path) -> Result<()> {
.with_context(|| "Failed to unmount the fuse session".to_string())?;

let _ = handle.join();

info!("Finished merging layers!");
Ok(())
}

impl FuseServer {
/// Run a loop to execute requests from the FUSE session
///
pub fn svc_loop(&mut self) -> Result<()> {
println!("entering server loop");
debug!("entering server loop");
loop {
let value = self
.ch
.get_request()
.with_context(|| "Failed to get message from fuse session".to_string())?;

if value.is_none() {
println!("fuse server exits");
debug!("fuse server exits");
break;
}

Expand All @@ -158,7 +168,7 @@ impl FuseServer {
break;
}
_ => {
print!("Handling fuse message failed");
warn!("Handling fuse message failed");
continue;
}
}
Expand Down
151 changes: 0 additions & 151 deletions src/fs-gen/src/image_loader.rs

This file was deleted.

Loading