Skip to content

Commit

Permalink
feat: add modules and helper scripts for working with tfgrid/zos micr…
Browse files Browse the repository at this point in the history
…ovms

* nixosModule.zosVmDir add `config.system.build.zosVmDir` which results
  in a directory that contains a rootfs, (uncompressed) kernel and
  initramfs. the resulting file structure can either be booted using
  virtiofsd + cloud-hypervisor or published to an s3 endpoint for
  consumption on tfgrid
* zos-vm-*: scripts to build, publish and local-boot zos microvm system images
  • Loading branch information
steveej committed Mar 27, 2024
1 parent ce275aa commit f7c78d4
Show file tree
Hide file tree
Showing 10 changed files with 707 additions and 32 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
.minio/
result*
.decrypted~keys.yaml
.storage
.direnv
zosVmDir**
95 changes: 95 additions & 0 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

88 changes: 56 additions & 32 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,17 @@
flake = false;
url = "github:steveej-forks/coturn/debug-cli-login";
};

nixos-generators = {
url = "github:nix-community/nixos-generators";
inputs.nixpkgs.follows = "nixpkgs";
};

threefold-rfs = {
url = "github:steveej-forks/threefold-rfs/sqlite-locking-issue";
inputs.nixpkgs.follows = "nixpkgs";
inputs.crane.follows = "crane";
};
};

outputs = inputs @ {
Expand All @@ -113,6 +124,7 @@
self',
inputs',
pkgs,
lib,
...
}: {
# Per-system attributes can be defined here. The self' and inputs'
Expand All @@ -124,40 +136,52 @@
nomadClientCert = ./secrets/nomad/cli/global-cli-nomad.pem;
in
pkgs.mkShell {
packages = [
pkgs.yq-go

inputs'.nixos-anywhere.packages.default

inputs'.sops-nix.packages.default
pkgs.ssh-to-age
pkgs.age
pkgs.age-plugin-yubikey
pkgs.sops

self'.packages.nomad

(pkgs.writeShellScriptBin "nomad-ui-proxy" (let
caddyfile = pkgs.writeText "caddyfile" ''
{
auto_https off
http_port 2016
}
localhost:2016 {
reverse_proxy ${nomadAddr} {
transport http {
tls_trusted_ca_certs ${nomadCaCert}
tls_client_auth ${nomadClientCert} {$NOMAD_CLIENT_KEY}
packages =
[
pkgs.yq-go

inputs'.nixos-anywhere.packages.default

inputs'.sops-nix.packages.default
pkgs.ssh-to-age
pkgs.age
pkgs.age-plugin-yubikey
pkgs.sops

# self'.packages.nomad

(pkgs.writeShellScriptBin "nomad-ui-proxy" (let
caddyfile = pkgs.writeText "caddyfile" ''
{
auto_https off
http_port 2016
}
localhost:2016 {
reverse_proxy ${nomadAddr} {
transport http {
tls_trusted_ca_certs ${nomadCaCert}
tls_client_auth ${nomadClientCert} {$NOMAD_CLIENT_KEY}
}
}
}
}
'';
in ''
${pkgs.caddy}/bin/caddy run --adapter caddyfile --config ${caddyfile}
''))
pkgs.caddy
];
'';
in ''
${pkgs.caddy}/bin/caddy run --adapter caddyfile --config ${caddyfile}
''))
pkgs.caddy

inputs'.threefold-rfs.packages.default
]
++ (
let
zosCmds = builtins.filter (pkg: null != (builtins.match "^zos-.*" pkg.name)) (builtins.attrValues self'.packages);
in
zosCmds
++ (lib.lists.flatten (builtins.map (cmd: cmd.nativeBuildInputs or []) zosCmds))
++ (lib.lists.flatten (builtins.map (cmd: cmd.buildInputs or []) zosCmds))
++ (lib.lists.flatten (builtins.map (cmd: cmd.runtimeInputs or []) zosCmds))
);

NOMAD_ADDR = nomadAddr;
NOMAD_CACERT = nomadCaCert;
Expand Down
47 changes: 47 additions & 0 deletions lib/make-system-directory.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
{ stdenv
, closureInfo
, pixz

, # The files and directories to be placed in the directory.
# This is a list of attribute sets {source, target} where `source'
# is the file system object (regular file or directory) to be
# grafted in the file system at path `target'.
contents

, # In addition to `contents', the closure of the store paths listed
# in `packages' are also placed in the Nix store of the tarball. This is
# a list of attribute sets {object, symlink} where `object' if a
# store path whose closure will be copied, and `symlink' is a
# symlink to `object' that will be added to the tarball.
storeContents ? [ ]

# Extra commands to be executed before archiving files
, extraCommands ? ""

# extra inputs
, extraInputs ? [ ]
}:

let
symlinks = map (x: x.symlink) storeContents;
objects = map (x: x.object) storeContents;
in

stdenv.mkDerivation {
name = "system-directory";
builder = ./make-system-directory.sh;
nativeBuildInputs = extraInputs;

inherit extraCommands;

# !!! should use XML.
sources = map (x: x.source) contents;
targets = map (x: x.target) contents;

# !!! should use XML.
inherit symlinks objects;

closureInfo = closureInfo {
rootPaths = objects;
};
}
53 changes: 53 additions & 0 deletions lib/make-system-directory.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
source $stdenv/setup

sources_=($sources)
targets_=($targets)

objects=($objects)
symlinks=($symlinks)

# Remove the initial slash from a path, since genisofs likes it that way.
stripSlash() {
res="$1"
if test "${res:0:1}" = /; then res=${res:1}; fi
}

# Add the individual files.
for ((i = 0; i < ${#targets_[@]}; i++)); do
stripSlash "${targets_[$i]}"
mkdir -p "$(dirname "$res")"
cp -a "${sources_[$i]}" "$res"
done

# Add the closures of the top-level store objects.
chmod +w .
mkdir -p nix/store
for i in $(<$closureInfo/store-paths); do
cp -a "$i" "${i:1}"
done

# TODO tar ruxo
# Also include a manifest of the closures in a format suitable for
# nix-store --load-db.
cp $closureInfo/registration nix-path-registration

# Add symlinks to the top-level store objects.
for ((n = 0; n < ${#objects[*]}; n++)); do
object=${objects[$n]}
symlink=${symlinks[$n]}
if test "$symlink" != "none"; then
mkdir -p $(dirname ./$symlink)
ln -s $object ./$symlink
fi
done

$extraCommands

rm env-vars

mkdir $out
cp -a --reflink=always * $out/

mkdir -p $out/nix-support
echo $system >$out/nix-support/system
echo "file system-directory $out" >$out/nix-support/hydra-build-products
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
config,
inputs,
self,
pkgs,
...
}: let
hostName = "tfgrid-devnet-vm0";
in {
imports = [
inputs.srvos.nixosModules.server
inputs.srvos.nixosModules.mixins-terminfo

inputs.sops-nix.nixosModules.sops

self.nixosModules.holo-users
../../nixos/shared.nix
../../nixos/shared-nix-settings.nix

self.nixosModules.zosVmDir
];

networking.hostName = hostName;

hostName = "TODO";

nix.settings.max-jobs = 8;

nix.settings.substituters = [
"https://holochain-ci.cachix.org"
];

nix.settings.trusted-public-keys = [
"holochain-ci.cachix.org-1:5IUSkZc0aoRS53rfkvH9Kid40NpyjwCMCzwRTXy+QN8="
];

system.stateVersion = "23.11";
}
Loading

0 comments on commit f7c78d4

Please sign in to comment.