From 02e9b7c377c09ca9a0367dc83cd55e568dc727a7 Mon Sep 17 00:00:00 2001 From: Niklas Saari Date: Thu, 26 Sep 2024 17:36:21 +0300 Subject: [PATCH] Make possible to provide outputdir, if not, use tempdir --- Cargo.toml | 2 + src/bin/cli.rs | 128 ++++++++++++++++++++++++------------------- src/build_process.rs | 16 +++--- src/moodle.rs | 2 +- 4 files changed, 84 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6caac4c..e5245ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,8 @@ aws-config = { version = "1.5.6", default-features = false, features = [ futures = "0.3.30" moodle-xml = "0.1.1" serde_json = "1.0.128" +once_cell = { version = "1.19.0", default-features = false } +tempfile = { version = "3.12.0", default-features = false } [dependencies.uuid] version = "1.9" features = [ diff --git a/src/bin/cli.rs b/src/bin/cli.rs index ef371ed..87f3d46 100644 --- a/src/bin/cli.rs +++ b/src/bin/cli.rs @@ -1,19 +1,26 @@ use ainigma::{ - build_process::{build_task, TaskBuildProcessOutput, OUTPUT_DIRECTORY}, + build_process::{build_task, TaskBuildProcessOutput}, config::{read_check_toml, ConfigError, ModuleConfiguration}, moodle::create_exam, storages::{CloudStorage, FileObjects, S3Storage}, }; use clap::{crate_description, Args, Parser, Subcommand}; +use once_cell::sync::Lazy; use std::{ - path::PathBuf, + path::{Path, PathBuf}, process::ExitCode, sync::{Arc, Mutex}, thread, }; + +use tempfile::TempDir; use tokio::runtime::Runtime; use uuid::Uuid; +// Lazily create a single global Tokio runtime +static RUNTIME: Lazy = + Lazy::new(|| Runtime::new().expect("Failed to create Tokio runtime")); + /// Autograder CLI Application #[derive(Parser, Debug)] #[command(name = "aínigma", version , about = "CLI for aínigma", long_about = crate_description!(), arg_required_else_help = true)] @@ -30,6 +37,10 @@ enum Commands { /// Build the specified tasks. #[command(arg_required_else_help = true)] Generate { + /// The output directory where the build files will be stored. If not set, using temporary directory. + /// Must exist and be writable if provided. + #[arg(short, long, value_name = "DIRECTORY")] + output_dir: Option, #[command(flatten)] selection: BuildSelection, /// Moodle subcommand is used to automatically upload the files into the cloud storage and then generate a Moodle exam. @@ -89,9 +100,7 @@ fn s3_upload( let mut tasks = Vec::with_capacity(files.len()); - let rt = Runtime::new().unwrap(); - - let health = rt.block_on(async { + let health = RUNTIME.block_on(async { match storage.health_check().await { Ok(_) => Ok(()), Err(error) => { @@ -141,11 +150,11 @@ fn s3_upload( }; tasks.push(future); } - let result = rt.block_on(async { futures::future::try_join_all(tasks).await }); + let result = RUNTIME.block_on(async { futures::future::try_join_all(tasks).await }); match result { Ok(files) => { if !config.deployment.upload.use_pre_signed { - let result = rt.block_on(async { storage.set_public_access().await }); + let result = RUNTIME.block_on(async { storage.set_public_access().await }); match result { Ok(_) => {} Err(error) => { @@ -183,6 +192,7 @@ fn main() -> std::process::ExitCode { } else { match &cli.command { Commands::Generate { + output_dir, selection, number, moodle, @@ -202,37 +212,53 @@ fn main() -> std::process::ExitCode { return ExitCode::FAILURE; } }; + let output_dir: Box> = match output_dir { + Some(output_dir) => { + if !output_dir.exists() { + tracing::error!( + "The provided output directory did not exist: {}", + output_dir.display() + ); + return ExitCode::FAILURE; + } else if !output_dir.is_dir() { + tracing::error!( + "The provided output directory is not a directory: {}", + output_dir.display() + ); + return ExitCode::FAILURE; + } else { + Box::new(output_dir) + } + } + None => { + let temp_dir = match TempDir::new() { + Ok(dir) => dir, + Err(error) => { + tracing::error!( + "Error when creating a temporal directory: {}", + error + ); + return ExitCode::FAILURE; + } + }; + tracing::info!( + "No output directory provided, using a temporal directory in path '{}'", + temp_dir.path().display() + ); + Box::new(temp_dir) + } + }; let outputs = match selection.task { - Some(ref task) => match parallel_task_build(&config, task, *number) { + Some(ref task) => match parallel_task_build( + &config, + task, + *number, + output_dir.as_ref().as_ref(), + ) { Ok(out) => out, Err(error) => { tracing::error!("Error when building the task: {}", error); - let task_config = config - .get_task_by_id(task) - .expect("No configuration found for the provided task."); - let output_dir = task_config.build.directory.join(OUTPUT_DIRECTORY); - if output_dir.exists() { - tracing::error!( - "Starting to remove all build-output files from folder: {}", - output_dir.display() - ); - let result = std::fs::remove_dir_all(&output_dir); - match result { - Ok(_) => { - tracing::info!( - "All build-output files removed from folder: {}", - output_dir.display() - ); - } - Err(error) => { - tracing::error!( - "Error when removing the build-output files: {}", - error - ); - } - } - } return ExitCode::FAILURE; } }, @@ -272,11 +298,16 @@ fn main() -> std::process::ExitCode { } }, }, - None => match parallel_task_build(&config, "test", 30) { - Ok(_) => (), - Err(error) => eprintln!("{}", error), - }, + None => { + match parallel_task_build(&config, "test", 30, output_dir.as_ref().as_ref()) + { + Ok(_) => (), + Err(error) => eprintln!("{}", error), + } + } } + // Ensure that possible temporal directory is removed at this point, not earlier + drop(output_dir); ExitCode::SUCCESS } Commands::Upload { check_bucket } => { @@ -313,6 +344,7 @@ fn parallel_task_build( config: &ModuleConfiguration, task: &str, number: usize, + output_dir: &Path, ) -> Result, ConfigError> { tracing::info!( "Building the task '{}' with the variation count {}", @@ -328,14 +360,16 @@ fn parallel_task_build( let mut handles = Vec::with_capacity(number); let course_config = Arc::new(config.clone()); let task_config = Arc::new(task_config); + let output_dir = Arc::new(output_dir.to_path_buf()); for i in 0..number { let courseconf = Arc::clone(&course_config); let taskconf = Arc::clone(&task_config); let outputs = Arc::clone(&all_outputs); + let outdir = Arc::clone(&output_dir); let handle = thread::spawn(move || { tracing::info!("Starting building the variant {}", i + 1); let uuid = Uuid::now_v7(); - let output = build_task(&courseconf, &taskconf, uuid); + let output = build_task(&courseconf, &taskconf, uuid, &outdir); match output { Ok(output) => { outputs @@ -364,7 +398,7 @@ fn parallel_task_build( } } else { let uuid = Uuid::now_v7(); - let outputs = build_task(config, &task_config, uuid).unwrap(); + let outputs = build_task(config, &task_config, uuid, output_dir).unwrap(); all_outputs.lock().unwrap().push(outputs); tracing::info!("Task '{}' build succesfully", &task); } @@ -375,22 +409,6 @@ fn parallel_task_build( Ok(vec) } -#[allow(dead_code)] -fn moodle_build( - config: ModuleConfiguration, - _week: Option, - task: Option<&str>, - number: usize, - _category: String, -) -> Result, ConfigError> { - match task { - Some(task) => parallel_task_build(&config, task, number), - None => { - todo!("Complete week build todo") - // let _result = parallel_build(path, week, task, number); - } - } -} #[cfg(test)] mod tests {} diff --git a/src/build_process.rs b/src/build_process.rs index 8d8b59b..d096c19 100644 --- a/src/build_process.rs +++ b/src/build_process.rs @@ -6,8 +6,6 @@ use uuid::Uuid; use crate::config::{BuildConfig, Builder, ModuleConfiguration, OutputKind, Task}; use crate::flag_generator::Flag; -pub const OUTPUT_DIRECTORY: &str = "output/"; - fn create_flag_id_pairs_by_task<'a>( task_config: &'a Task, module_config: &'a ModuleConfiguration, @@ -155,21 +153,23 @@ pub fn build_task( module_config: &ModuleConfiguration, task_config: &Task, uuid: Uuid, + output_directory: &Path, ) -> Result> { let (flags, mut build_envs) = create_flag_id_pairs_by_task(task_config, module_config, uuid); match task_config.build.builder { Builder::Shell(ref entrypoint) => { - let build_output = Path::new(OUTPUT_DIRECTORY).join(uuid.to_string()); - let builder_relative_dir = Path::new(&task_config.build.directory).join(&build_output); + let builder_output_dir = output_directory + .join(uuid.to_string()) + .join(&task_config.id); tracing::debug!( "Running shell command: {} with flags: {:?} in directory: {}", entrypoint.entrypoint, build_envs, - &builder_relative_dir.display() + &builder_output_dir.display() ); // Create all required directories in the path - match fs::create_dir_all(&builder_relative_dir) { + match fs::create_dir_all(&builder_output_dir) { Ok(_) => (), Err(e) => { tracing::error!( @@ -184,7 +184,7 @@ pub fn build_task( // This means that output directory should relatively referenced based on the CWD of this program build_envs.insert( "OUTPUT_DIR".to_string(), - build_output.to_str().unwrap_or_default().to_string(), + builder_output_dir.to_str().unwrap_or_default().to_string(), ); let output = std::process::Command::new("sh") .arg(&entrypoint.entrypoint) @@ -207,7 +207,7 @@ pub fn build_task( let mut outputs = Vec::with_capacity(task_config.build.output.len()); if output.status.success() { for output in &task_config.build.output { - let path = builder_relative_dir + let path = builder_output_dir .join(output.kind.get_filename()) .canonicalize()?; match fs::metadata(&path) { diff --git a/src/moodle.rs b/src/moodle.rs index 61d1ad6..dc5ebf2 100644 --- a/src/moodle.rs +++ b/src/moodle.rs @@ -25,7 +25,7 @@ pub fn create_exam( let mut lines: Vec = reader.lines().collect::>()?; lines.push("".to_string()); - lines.push("

Please, see the download links below. Exam questions are randomised and the link is different if you retry the exam.".to_string()); + lines.push("

Please, see the download links below. Exam questions are randomised and the links are different if you retry the exam.".to_string()); lines.push("
".to_string()); lines.push( "
"