Skip to content

Commit

Permalink
Merge pull request #131 from holochain/s3-storage-for-flists-minio
Browse files Browse the repository at this point in the history
feat: set up x64-linux-dev-01 as a (tfgrid flists) development server, including a persistent s3 storage for flist hosting
  • Loading branch information
steveej authored Jul 2, 2024
2 parents 846d457 + ebd5cf2 commit 19bbbfc
Show file tree
Hide file tree
Showing 18 changed files with 490 additions and 113 deletions.
22 changes: 21 additions & 1 deletion .sops.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
keys:
- &steveej 6F7069FE6B96E894E60EC45C6EEFA706CB17E89B
- &jost-s D299483493EAE6B2B3D892B6D33548FA55FF167F
- &dev age1fnmdutanvfsrhadap3qsmncjfa85x82qy8svy98ma4p37dglq45stcwk28
- &dweb-reverse-proxy age1ygzy9clj0xavlmau0ham7j5nw8yy4z0q8hvkfpdgwc4fcr8nufpqrdxgvx
- &linux-builder-01 age1kxkr407jz77ljrhgsfwfmv2yvqjprc6unvx389xp2f48xj8r0vqq2wew5r
- &x64-linux-dev-01 age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9
Expand All @@ -26,6 +27,12 @@ creation_rules:
key_groups:
- age:
- *linux-builder-01
- path_regex: ^secrets/x64-linux-dev-01/[^/]+$
key_groups:
- pgp:
- *steveej
age:
- *x64-linux-dev-01
- path_regex: ^secrets/nomad/.+$
key_groups:
- pgp:
Expand All @@ -42,4 +49,17 @@ creation_rules:
key_groups:
- pgp:
- *steveej

- path_regex: ^secrets/dev/.+$
key_groups:
- pgp:
- *steveej
age:
- *dev
- *x64-linux-dev-01
- path_regex: ^secrets/minio/.+$
key_groups:
- pgp:
- *steveej
age:
- *dev
- *x64-linux-dev-01
6 changes: 3 additions & 3 deletions flake.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

38 changes: 30 additions & 8 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@
pkgs.caddy

inputs'.threefold-rfs.packages.default
pkgs.minio-client

pkgs.jq
pkgsPulumi.pulumictl
Expand All @@ -212,14 +213,35 @@
NOMAD_CACERT = nomadCaCert;
NOMAD_CLIENT_CERT = nomadClientCert;

shellHook = ''
set -x
REPO_SECRETS_DIR="''${HOME:?}/.holochain-infra-secrets"
mkdir -p ''${REPO_SECRETS_DIR}
chmod 700 ''${REPO_SECRETS_DIR}
export NOMAD_CLIENT_KEY="''${REPO_SECRETS_DIR}/global-cli-nomad-key";
sops -d secrets/nomad/cli/keys.yaml | yq '.global-cli-nomad-key' > ''${NOMAD_CLIENT_KEY:?}
'';
shellHook = let
devMinioOsConfig = self.nixosConfigurations.x64-linux-dev-01.config;
in
''
if sops -d secrets/nomad/cli/keys.yaml 2>&1 >/dev/null; then
REPO_SECRETS_DIR="''${HOME:?}/.holochain-infra-secrets"
mkdir -p ''${REPO_SECRETS_DIR}
chmod 700 ''${REPO_SECRETS_DIR}
export NOMAD_CLIENT_KEY="''${REPO_SECRETS_DIR}/global-cli-nomad-key";
sops -d secrets/nomad/cli/keys.yaml | yq '.global-cli-nomad-key' > ''${NOMAD_CLIENT_KEY:?}
fi
''
+ (let
minioUserPass = ''''${MINIO_ROOT_USER}:''${MINIO_ROOT_PASSWORD}'';
minioDevHost = devMinioOsConfig.services.devMinio.s3Domain + ":443";
minioDevLocalHost = "127.0.0.1:${builtins.toString devMinioOsConfig.services.devMinio.listenPort}";
minioRegion = devMinioOsConfig.services.devMinio.region;
in ''
if sops -d secrets/minio/server.yaml 2>&1 >/dev/null; then
source <(sops -d secrets/minio/server.yaml | yq '.minio_root_credentials')
export MC_HOST_devminio_local="http://${minioUserPass}@${minioDevLocalHost}";
export MC_HOST_devminio="https://${minioUserPass}@${minioDevHost}"
export RFS_HOST_devminio_region="${minioRegion}"
export RFS_HOST_devminio_local="s3://${minioUserPass}@${minioDevLocalHost}"
export RFS_HOST_devminio="s3s://${minioUserPass}@${minioDevHost}"
fi
'');
};

packages =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -183,8 +183,9 @@ in {
hackathon.events.${fqdn2domain}. A 10.1.3.37
amsterdam2023.events.${fqdn2domain}. A 10.1.3.187
sj-bm-hostkey0.dev.${fqdn2domain}. A 185.130.224.33
x64-linux-dev-01.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
x64-linux-dev-01.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
s3.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
s3-console.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
turn-0.${fqdn2domain}. A ${self.nixosConfigurations.turn-0.config.services.holochain-turn-server.address}
signal-0.${fqdn2domain}. A ${self.nixosConfigurations.turn-0.config.services.tx5-signal-server.address}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
{
config,
inputs,
self,
pkgs,
lib,
config,
...
}: {
}: let
in {
imports = [
inputs.disko.nixosModules.disko
inputs.srvos.nixosModules.server
inputs.srvos.nixosModules.hardware-hetzner-online-amd
inputs.srvos.nixosModules.roles-nix-remote-builder
self.nixosModules.holo-users
self.nixosModules.holo-users-interactive

self.nixosModules.nix-build-distributor

Expand All @@ -20,6 +22,23 @@
../../nixos/shared.nix
../../nixos/shared-nix-settings.nix
../../nixos/shared-linux.nix

{
home-manager.users.dev = {pkgs, ...}: {
home.packages = [
# additional packages for this user go here
];
};
}

../../nixos/dev-minio.nix
{
services.devMinio.enable = true;
}
];

nix.settings.system-features = [
"big-parallel"
];

networking = {
Expand Down
101 changes: 93 additions & 8 deletions modules/flake-parts/nixosModules.holo-users.nix
Original file line number Diff line number Diff line change
@@ -1,15 +1,100 @@
{
self,
inputs,
lib,
...
}: {
flake.nixosModules.holo-users = {
users.users.root.openssh.authorizedKeys = {
keyFiles =
lib.attrValues
(lib.filterAttrs (name: _: lib.hasPrefix "keys_" name) inputs);
keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICHujII5RAwfEXNBYxKhWv2Wx/oHeHUTc8CACZ3M5W3p neonphog@gmail.com"
}: let
mkAuthorizedKeys = {...}: {
keyFiles =
lib.attrValues
(lib.filterAttrs (name: _: lib.hasPrefix "keys_" name) inputs);
keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICHujII5RAwfEXNBYxKhWv2Wx/oHeHUTc8CACZ3M5W3p neonphog@gmail.com"
];
};
in {
flake.nixosModules.holo-users = {config, ...}: {
users.mutableUsers = false;
users.users.root.openssh.authorizedKeys = mkAuthorizedKeys {};
};

flake.nixosModules.holo-users-interactive = {config, ...}: {
imports = [
inputs.home-manager.nixosModules.home-manager
];

# a generic dev user that can be used to have per-host home-manager environments for it.
# this adds no risk since all potential users already have access to the root account via their SSH credentials.
users.users.dev = {
home = "/home/dev";
extraGroups = ["wheel"];
openssh.authorizedKeys = mkAuthorizedKeys {};
isNormalUser = true;
createHome = true;
};

sops.secrets.dev-age-key = {
sopsFile = self + "/secrets/dev/secrets.yaml";
owner = "dev";
};
home-manager = {
useGlobalPkgs = true;
useUserPackages = true;

sharedModules = [
inputs.sops-nix.homeManagerModules.sops
];
users.dev = {pkgs, ...}: {
# Home Manager needs a bit of information about you and the
# paths it should manage.
home.username = "dev";
home.homeDirectory = "/home/dev";

home.packages = [
pkgs.coreutils
pkgs.neovim
];

programs.bash.enable = true;
programs.bash.sessionVariables.SOPS_AGE_KEY_FILE = config.sops.secrets.dev-age-key.path;

programs.direnv.enable = true;
# TODO: enable this once home-manager is bumped to >= release-24.05
# programs.nix-direnv.enable = true;

# This value determines the Home Manager release that your
# configuration is compatible with. This helps avoid breakage
# when a new Home Manager release introduces backwards
# incompatible changes.
#
# You can update Home Manager without changing this value. See
# the Home Manager release notes for a list of state version
# changes in each release.
home.stateVersion = "23.11";

# Let Home Manager install and manage itself.
programs.home-manager.enable = true;

sops = {
age.keyFile = config.sops.secrets.dev-age-key.path;
defaultSopsFile = self + "/secrets/dev/secrets.yaml";
};
};
};

security.sudo = {
enable = true;
execWheelOnly = true;
extraRules = [
{
groups = ["wheel"];
commands = [
{
command = "ALL";
options = ["NOPASSWD"];
}
];
}
];
};
};
Expand Down
50 changes: 23 additions & 27 deletions modules/flake-parts/packages.zos-utils.nix
Original file line number Diff line number Diff line change
Expand Up @@ -31,29 +31,16 @@
'';
};

# TODO: automate proper minio hosting. this is exemplary only and requires imperative setup of minio
zos-vm-serve-s3 = pkgs.writeShellApplication {
name = "zos-vm-serve-s3";
runtimeInputs = [
pkgs.minio
];
text = ''
set -ueE -o pipefail
cd .minio
env \
MINIO_ROOT_USER=minioadmin \
MINIO_ROOT_PASSWORD="$(cat minioadmin.key)" \
minio server --console-address ":9001" storage
'';
};

zos-vm-publish-s3 = let
s3BaseUrl = "sj-bm-hostkey0.dev.infra.holochain.org";
s3ListenUrl = "${s3BaseUrl}:9000";
s3HttpUrl = "https://${s3BaseUrl}/s3";
# TODO: document these: explain the relationship to the variables in the devShell's shellHook; if viable give them a common source of truth
s3BaseUrl = "dev.infra.holochain.org";
s3HttpUrl = "https://s3.${s3BaseUrl}";

# TODO: programmatically ensure this exists
s3Bucket = "tfgrid-eval";

# TODO: this is faster however restricts publishing to the server itself
s3Alias = "devminio_local";
in
pkgs.writeShellApplication {
name = "zos-vm-publish-s3";
Expand All @@ -73,12 +60,16 @@
mkdir -p "$workDir"
cd "$workDir"
s3_remote="''${MC_HOST_devminio_local:?}"
s3_remote="''${s3_remote/http:\/\//}"
# mc rm --recursive --force localhost/${s3Bucket} || echo removal failed
env RUST_MIN_STACK=8388608 \
rfs pack -m result.fl -s s3://minioadmin:"$(cat ../../.minio/minioadmin.key)"@${s3ListenUrl}/${s3Bucket}\?region=us-east-1 "$rootfs/" | tee rfs-pack.log
rfs pack -m result.fl -s "''${RFS_HOST_devminio:?}/${s3Bucket}?region=''${RFS_HOST_devminio_region:?}" "$rootfs/" | tee rfs-pack.log
mc cp result.fl ${s3Alias}/${s3Bucket}/"$rootfsBase".fl
# TODO: document or automate setting up the alias "localhost"
mc cp result.fl localhost/${s3Bucket}/"$rootfsBase".fl
# the final URL doesn't have the bucket name as it's implied as the default bucket.
echo ${s3HttpUrl}/${s3Bucket}/"$rootfsBase".fl > public-url
touch published
Expand Down Expand Up @@ -208,9 +199,12 @@
kernel="$rootfs/boot/vmlinuz"
initram="$rootfs/boot/initrd.img"
# FIXME: can't handle path ending in '/'
workDir="$rootfs.work"
mountDir="$workDir/mnt"
mkdir -p "$mountDir"
# set it to read-only by default. the mount will be writable.
chmod 440 "$mountDir"
socket="$workDir/virtiofs.sock"
Expand All @@ -219,7 +213,9 @@
exit 1
}
rfs mount -m "$workDir"/result.fl "$mountDir" > "$workDir"/rfs_mount.log 2>&1 &
# FIXME: check whether the mount was successful
# FIXME: don't rely on sudo
sudo rfs mount -m "$workDir"/result.fl "$mountDir" 2>&1 | tee "$workDir"/rfs_mount.log &
mountpid="$!"
sleep 3
Expand All @@ -243,8 +239,8 @@
sudo kill "$fspid"
rm -rf "$socket"
kill "$mountpid"
umount --lazy "$mountDir"
sudo kill "$mountpid"
sudo umount --lazy "$mountDir"
rmdir "$mountDir"
)
}
Expand Down
Loading

0 comments on commit 19bbbfc

Please sign in to comment.