diff --git a/Cargo.lock b/Cargo.lock index 4822897..95004ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,7 +47,7 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -57,7 +57,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -235,6 +235,22 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" + [[package]] name = "generic-array" version = "0.14.7" @@ -325,6 +341,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "log" version = "0.4.22" @@ -429,6 +451,7 @@ dependencies = [ "env_logger", "log", "serde_json", + "tempfile", ] [[package]] @@ -500,6 +523,19 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rustix" +version = "0.38.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "ryu" version = "1.0.18" @@ -569,6 +605,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "thiserror" version = "1.0.64" @@ -683,6 +732,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" diff --git a/applications/postbuildstepper/Cargo.toml b/applications/postbuildstepper/Cargo.toml index 54c359e..55f6766 100644 --- a/applications/postbuildstepper/Cargo.toml +++ b/applications/postbuildstepper/Cargo.toml @@ -11,3 +11,4 @@ config = { version = "0.14.0", features = ["json", "json5", "toml"] } env_logger = "0.11.5" log = "0.4.22" serde_json = "1.0.132" +tempfile = "3.13.0" diff --git a/applications/postbuildstepper/src/main.rs b/applications/postbuildstepper/src/main.rs index a2cae78..4b58308 100644 --- a/applications/postbuildstepper/src/main.rs +++ b/applications/postbuildstepper/src/main.rs @@ -1,6 +1,12 @@ -use anyhow::{bail, Context}; -use log::{debug, info, trace}; -use std::collections::{HashMap, HashSet}; +use anyhow::{bail, Context, Ok}; +use log::{debug, info, trace, warn}; +use std::{ + collections::{HashMap, HashSet}, + io::Write, + path::PathBuf, + process::Stdio, +}; +use tempfile::{tempfile, NamedTempFile, TempPath}; /* set -Eu -o pipefail @@ -25,12 +31,130 @@ fn main() -> anyhow::Result<()> { check_owners(&env_vars)?; - debug!("TODO: if org is holo-host sign store path recursively"); - debug!("TODO: if org is holo-host push to s3"); + let (signing_key_file, s3_credentials_profile) = + if let Some(skf) = may_get_signing_key_and_s3_credentials(&env_vars)? { + skf + } else { + warn!("got no signing/uploading credentials, exiting."); + return Ok(()); + }; + let signing_key_file_path = signing_key_file.path().to_str().ok_or_else(|| { + anyhow::anyhow!( + "could not convert {} (lossy) to string", + signing_key_file.path().to_string_lossy() + ) + })?; + + let store_path = "TODO"; + + // sign the store path + { + let mut cmd = std::process::Command::new("nix"); + cmd.args([ + "store", + "sign", + "--recursive", + &format!("--key-file={signing_key_file_path}"), + store_path, + ]) + // let stdio go through so it's visible in the logs + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + cmd.spawn().context(format!("running {cmd:#?}"))?; + + info!("successfully signed store path {store_path}"); + } + + // copy the store path + { + let mut cmd = std::process::Command::new("nix"); + cmd.args(["copy", "--to", copy_destination, store_path]) + // let stdio go through so it's visible in the logs + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + cmd.spawn().context(format!("running {cmd:#?}"))?; + + debug!("TODO: push to s3"); + } Ok(()) } +/// Evaluates the project org and accordingly returns a signing key. +fn may_get_signing_key_and_s3_credentials( + env_vars: &HashMap, +) -> Result, anyhow::Error> { + let (org, _) = { + // example var: 'PROP_project=holochain/holochain-infra + + // FIXME: create a constant or config value for this + let var = "PROP_project"; + let value = env_vars + .get(var) + .context(format!("looking up {var} in {env_vars:#?}"))?; + + if let Some(split) = value.split_once("/") { + split + } else { + bail!("couldn't parse project {value}"); + } + }; + + let wrap_secret_in_tempfile = |s: &str| { + let mut tempfile = NamedTempFile::new()?; + tempfile.write_all(s.as_bytes())?; + Ok(tempfile) + }; + + // FIXME: remove this? it's used for testing purposes + let override_holo_sign = { + // example var: 'PROP_project=holochain/holochain-infra + + // FIXME: create a constant or config value for this + let var = "PROP_attr"; + + let value = env_vars + .get(var) + .context(format!("looking up {var} in {env_vars:#?}"))?; + + value == "aarch64-darwin.pre-commit-check" + }; + + if org.to_lowercase() == "holo-host" || override_holo_sign { + info!("TODO: sign with holo's key"); + + // FIXME: create a constant or config value for this + // TODO: use the secret key instead + let var = "SECRET_cacheHoloHost2public"; + let value = env_vars + .get(var) + .context(format!("looking up {var} in {env_vars:#?}"))?; + + let copy_destination = { + // FIXME: create a config map for these + let s3_bucket = "cache.holo.host"; + let s3_endpoint = "s3.wasabisys.com"; + let s3_profile = "cache-holo-host-s3-wasabi"; + + // TODO: is the secret-key still needed when `nix sign` is performed separately? + // &secret-key=/var/lib/hydra/queue-runner/keys/${signingKeyName}/secret + // TODO: will this accumulate a cache locally that needs maintenance? + format!("s3://{s3_bucket}?endpoint=${s3_endpoint}&log-compression=br&ls-compression=br¶llel-compression=1&write-nar-listing=1&profile={s3_profile}") + }; + + Ok(Some((wrap_secret_in_tempfile(value)?, copy_destination))) + } else if org.to_lowercase() == "holochain" { + info!("TODO: sign with holochain's key"); + + Ok(None) + } else { + warn!("unknown org: {org}"); + Ok(None) + } +} + fn check_owners(env_vars: &HashMap) -> Result<(), anyhow::Error> { let trusted_owners = HashSet::::from_iter(["steveej"].map(ToString::to_string)); let owners: HashSet = { diff --git a/modules/flake-parts/nixosConfigurations.buildbot-nix-0/configuration.nix b/modules/flake-parts/nixosConfigurations.buildbot-nix-0/configuration.nix index c0d3efb..a2c6648 100644 --- a/modules/flake-parts/nixosConfigurations.buildbot-nix-0/configuration.nix +++ b/modules/flake-parts/nixosConfigurations.buildbot-nix-0/configuration.nix @@ -306,6 +306,11 @@ /* replicate this hydra config + wasabiBucket = "cache.holo.host"; + wasabiEndpoint = "s3.wasabisys.com"; + # TODO: bring into proper hydra module + signingKeyName = "cache.holo.host-2"; + ```nix binary_cache_public_uri = https://cache.holo.host log_prefix = https://cache.holo.host/