diff --git a/Cargo.lock b/Cargo.lock index 95004ba..9b22392 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,6 +101,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cc" +version = "1.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -312,6 +323,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "json5" version = "0.4.1" @@ -397,6 +417,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d61c5ce1153ab5b689d0c074c4e7fc613e942dfb7dd9eea5ab202d2ad91fe361" +[[package]] +name = "pcre2" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be55c43ac18044541d58d897e8f4c55157218428953ebd39d86df3ba0286b2b" +dependencies = [ + "libc", + "log", + "pcre2-sys", +] + +[[package]] +name = "pcre2-sys" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "550f5d18fb1b90c20b87e161852c10cde77858c3900c5059b5ad2a1449f11d8a" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "pest" version = "2.7.14" @@ -442,6 +484,12 @@ dependencies = [ "sha2", ] +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + [[package]] name = "postbuildstepper" version = "0.1.0" @@ -450,6 +498,7 @@ dependencies = [ "config", "env_logger", "log", + "pcre2", "serde_json", "tempfile", ] @@ -594,6 +643,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "syn" version = "2.0.80" diff --git a/applications/postbuildstepper/Cargo.toml b/applications/postbuildstepper/Cargo.toml index 55f6766..e6422a5 100644 --- a/applications/postbuildstepper/Cargo.toml +++ b/applications/postbuildstepper/Cargo.toml @@ -10,5 +10,6 @@ anyhow = "1.0.90" config = { version = "0.14.0", features = ["json", "json5", "toml"] } env_logger = "0.11.5" log = "0.4.22" +pcre2 = "0.2.9" serde_json = "1.0.132" tempfile = "3.13.0" diff --git a/applications/postbuildstepper/src/main.rs b/applications/postbuildstepper/src/main.rs index 4b58308..61ea0b2 100644 --- a/applications/postbuildstepper/src/main.rs +++ b/applications/postbuildstepper/src/main.rs @@ -1,12 +1,6 @@ -use anyhow::{bail, Context, Ok}; -use log::{debug, info, trace, warn}; -use std::{ - collections::{HashMap, HashSet}, - io::Write, - path::PathBuf, - process::Stdio, -}; -use tempfile::{tempfile, NamedTempFile, TempPath}; +use anyhow::Ok; +use log::{info, trace, warn}; +use std::collections::HashMap; /* set -Eu -o pipefail @@ -20,20 +14,18 @@ echo ''${SECRET_cacheHoloHost2public} > public-key cat public-key */ - fn main() -> anyhow::Result<()> { env_logger::builder() .filter_level(log::LevelFilter::Debug) .init(); - let env_vars = HashMap::::from_iter(std::env::vars()); - debug!("env vars: {env_vars:#?}"); + let build_info = business::BuildInfo::from_env(); - check_owners(&env_vars)?; + let _ = business::check_owners(build_info.try_owners()?); - let (signing_key_file, s3_credentials_profile) = - if let Some(skf) = may_get_signing_key_and_s3_credentials(&env_vars)? { - skf + let (signing_key_file, copy_destination) = + if let Some(info) = business::may_get_signing_key_and_copy_destination(&build_info)? { + info } else { warn!("got no signing/uploading credentials, exiting."); return Ok(()); @@ -45,135 +37,240 @@ fn main() -> anyhow::Result<()> { ) })?; - let store_path = "TODO"; + // TODO: read the attribute name from the environment + let store_path = "./result"; // sign the store path - { - let mut cmd = std::process::Command::new("nix"); - cmd.args([ - "store", - "sign", - "--recursive", - &format!("--key-file={signing_key_file_path}"), - store_path, - ]) - // let stdio go through so it's visible in the logs - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()); - - cmd.spawn().context(format!("running {cmd:#?}"))?; - - info!("successfully signed store path {store_path}"); - } + util::nix_cmd_helper(&[ + "store", + "sign", + "--verbose", + "--recursive", + "--key-file", + signing_key_file_path, + store_path, + ])?; + info!("successfully signed store path {store_path}"); // copy the store path - { + util::nix_cmd_helper(&["copy", "--verbose", "--to", ©_destination, store_path])?; + info!("successfully pushed store path {store_path}"); + + Ok(()) +} + +mod util { + use std::process::Stdio; + + use anyhow::{bail, Context}; + + pub(crate) fn nix_cmd_helper(args: &[&str]) -> anyhow::Result<()> { let mut cmd = std::process::Command::new("nix"); - cmd.args(["copy", "--to", copy_destination, store_path]) - // let stdio go through so it's visible in the logs + cmd.args(args) + // pass stdio through so it becomes visible in the log .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); - cmd.spawn().context(format!("running {cmd:#?}"))?; + let context = format!("running {cmd:#?}"); - debug!("TODO: push to s3"); - } + let mut spawned = cmd.spawn().context(context.clone())?; + let finished = spawned.wait().context(context.clone())?; + if !finished.success() { + bail!("{context} failed."); + } - Ok(()) + Ok(()) + } } -/// Evaluates the project org and accordingly returns a signing key. -fn may_get_signing_key_and_s3_credentials( - env_vars: &HashMap, -) -> Result, anyhow::Error> { - let (org, _) = { +mod business { + use std::{ + collections::{HashMap, HashSet}, + io::Write, + }; + + use anyhow::{bail, Context, Result}; + use log::{debug, info, trace, warn}; + use tempfile::NamedTempFile; + + #[derive(Debug)] + pub(crate) struct BuildInfo(HashMap); + + // FIXME: is hardocing these in a type and functions sustainable, or is a config map appropriate? + impl BuildInfo { // example var: 'PROP_project=holochain/holochain-infra + pub(crate) fn from_env() -> Self { + let env_vars = HashMap::::from_iter(std::env::vars()); - // FIXME: create a constant or config value for this - let var = "PROP_project"; - let value = env_vars - .get(var) - .context(format!("looking up {var} in {env_vars:#?}"))?; + let new_self = Self(env_vars); + trace!("env vars: {new_self:#?}"); - if let Some(split) = value.split_once("/") { - split - } else { - bail!("couldn't parse project {value}"); + new_self + } + fn get(&self, var: &str) -> Result<&String> { + self.0 + .get(var) + .context(format!("looking up {var} in {self:#?}")) } - }; - - let wrap_secret_in_tempfile = |s: &str| { - let mut tempfile = NamedTempFile::new()?; - tempfile.write_all(s.as_bytes())?; - Ok(tempfile) - }; - // FIXME: remove this? it's used for testing purposes - let override_holo_sign = { - // example var: 'PROP_project=holochain/holochain-infra + pub(crate) fn try_owners(&self) -> Result> { + let value = self.get("PROP_owners")?; + let vec: Vec = serde_json::from_str(&value.replace("\'", "\"")) + .context(format!("parsing {value:?} as JSON"))?; - // FIXME: create a constant or config value for this - let var = "PROP_attr"; + Ok(HashSet::from_iter(vec)) + } + pub(crate) fn try_org_repo(&self) -> Result<(&str, &str)> { + let value = self.get("PROP_project")?; + + if let Some(split) = value.split_once("/") { + Ok(split) + } else { + bail!("couldn't parse project {value}"); + } + } - let value = env_vars - .get(var) - .context(format!("looking up {var} in {env_vars:#?}"))?; + pub(crate) fn try_attr(&self) -> Result<&String> { + self.get("PROP_attr") + } - value == "aarch64-darwin.pre-commit-check" - }; + pub(crate) fn try_attr_name(&self) -> Result<&str> { + let attr = self.get("PROP_attr")?; - if org.to_lowercase() == "holo-host" || override_holo_sign { - info!("TODO: sign with holo's key"); - - // FIXME: create a constant or config value for this - // TODO: use the secret key instead - let var = "SECRET_cacheHoloHost2public"; - let value = env_vars - .get(var) - .context(format!("looking up {var} in {env_vars:#?}"))?; - - let copy_destination = { - // FIXME: create a config map for these - let s3_bucket = "cache.holo.host"; - let s3_endpoint = "s3.wasabisys.com"; - let s3_profile = "cache-holo-host-s3-wasabi"; - - // TODO: is the secret-key still needed when `nix sign` is performed separately? - // &secret-key=/var/lib/hydra/queue-runner/keys/${signingKeyName}/secret - // TODO: will this accumulate a cache locally that needs maintenance? - format!("s3://{s3_bucket}?endpoint=${s3_endpoint}&log-compression=br&ls-compression=br¶llel-compression=1&write-nar-listing=1&profile={s3_profile}") - }; + attr.split_once(".") + .ok_or_else(|| anyhow::anyhow!("{attr} does not contain a '.'")) + .map(|r| r.1) + } + } - Ok(Some((wrap_secret_in_tempfile(value)?, copy_destination))) - } else if org.to_lowercase() == "holochain" { - info!("TODO: sign with holochain's key"); + /// Verifies that the build current owners are trusted. + // FIXME: make trusted owners configurable + pub(crate) fn check_owners(owners: HashSet) -> anyhow::Result<()> { + const TRUSTED_OWNERS: [&str; 1] = ["steveej"]; + let trusted_owners = HashSet::::from_iter(TRUSTED_OWNERS.map(ToString::to_string)); + let owner_is_trusted = owners.is_subset(&trusted_owners); + if !owner_is_trusted { + bail!("{owners:?} are *NOT* trusted!"); + } + info!("owners {owners:?} are trusted! proceeding."); - Ok(None) - } else { - warn!("unknown org: {org}"); - Ok(None) + Ok(()) } -} -fn check_owners(env_vars: &HashMap) -> Result<(), anyhow::Error> { - let trusted_owners = HashSet::::from_iter(["steveej"].map(ToString::to_string)); - let owners: HashSet = { - let var = "PROP_owners"; + /// Evaluates the project org and accordingly returns a signing key. + pub(crate) fn may_get_signing_key_and_copy_destination( + build_info: &BuildInfo, + ) -> anyhow::Result> { + let (org, repo) = build_info.try_org_repo()?; - let value = env_vars - .get(var) - .context(format!("looking up {var} in {env_vars:#?}"))?; + let wrap_secret_in_tempfile = |s: &str| -> anyhow::Result<_> { + let mut tempfile = NamedTempFile::new()?; + tempfile.write_all(s.as_bytes())?; + Ok(tempfile) + }; - let vec: Vec = serde_json::from_str(&value.replace("\'", "\"")) - .context(format!("parsing {value:?} as JSON"))?; + let attr_name = build_info.try_attr_name()?; + + // FIXME: remove this? it's used for testing purposes + let override_holo_sign = + { org == "holochain" && repo == "holochain-infra" && attr_name == "pre-commit-check" }; + debug!("override_holo_sign? {override_holo_sign:#?}"); + + let maybe_data = if org.to_lowercase() == "holo-host" || override_holo_sign { + // FIXME: create a constant or config value for this + let secret = build_info.get("SECRET_cacheHoloHost2secret")?; + + let copy_destination = { + // FIXME: create a config map for all the below + + // TODO: is the secret-key still needed when `nix sign` is performed separately? &secret-key=/var/lib/hydra/queue-runner/keys/${signingKeyName}/secret + // TODO: will this accumulate a cache locally that needs maintenance? + + let s3_bucket = "cache.holo.host"; + let s3_endpoint = "s3.wasabisys.com"; + let s3_profile = "cache-holo-host-s3-wasabi"; + + format!("s3://{s3_bucket}?") + + &[ + vec![ + format!("endpoint={s3_endpoint}"), + format!("profile={s3_profile}"), + ], + [ + "log-compression=br", + "ls-compression=br", + "parallel-compression=1", + "write-nar-listing=1", + ] + .into_iter() + .map(ToString::to_string) + .collect(), + ] + .concat() + .join("&") + }; + + Some((wrap_secret_in_tempfile(secret)?, copy_destination)) + } else if org.to_lowercase() == "holochain" { + info!("TODO: sign with holochain's key"); + None + } else { + warn!("unknown org: {org}"); + None + }; - HashSet::from_iter(vec) - }; - let owner_is_trusted = owners.is_subset(&trusted_owners); - if !owner_is_trusted { - bail!("{owners:#?} are *NOT* trusted!"); + let data = if let Some(data) = maybe_data { + data + } else { + return Ok(None); + }; + + let is_match_lossy = |re: &str, s: &str| { + let is_match = pcre2::bytes::Regex::new(re) + .map_err(|e| { + log::error!("error parsing {re} as regex: {e}"); + }) + .and_then(|re| { + re.is_match(s.as_bytes()).map_err(|e| { + log::error!("error parsing {re:?} as regex: {e}"); + }) + }) + .unwrap_or(false); + + debug!("{re} matched {s}: {is_match}"); + + is_match + }; + + // pass and exclude filter for well-known attrs + // FIXME: create a config map for this + const ATTR_PASS_FILTER_RE: &str = ".*pre-commit-check"; + // FIXME: create a config map for this + const ATTR_EXCLUDE_FILTER_RE: &str = "tests-.*"; + let attr = build_info.try_attr()?; + let pass = is_match_lossy(ATTR_PASS_FILTER_RE, attr) + && !is_match_lossy(ATTR_EXCLUDE_FILTER_RE, attr); + if !pass { + warn!("excluding '{attr}'."); + return Ok(None); + } + + Ok(Some(data)) } - info!("{owners:#?} are trusted! proceeding."); +} - Ok(()) +#[cfg(test)] +mod tests { + // TODO + + /* + initial testing done manually using + + env \ + PROP_owners="['steveej']" \ + PROP_project="holochain/holochain-infra" \ + PROP_attr="aarch64-linux.pre-commit-check" \ + SECRET_cacheHoloHost2secret="testing:27QUePIhJDF8BK3l3R8qP78Id9LeRsrp/ScD84ulL7BVv0McPC8+p+9zgvtsNzvCubLzyQNzpjIshSqoC7XmEQ==" \ + nix run .\#postbuildstepper + */ }