From c5b5e6cd7511e318b51c4582023e82ce453f6305 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Wed, 26 Jun 2024 17:53:03 +0200 Subject: [PATCH 1/4] feat: add holo-users-interactive with "dev" user this user is meant ofr interactive sessions on servers with a non-root account for development purposes. adds an age key for secrets that are required for services during development. update home-manager module as well. --- .sops.yaml | 9 +- flake.lock | 6 +- .../configuration.nix | 19 +++- .../flake-parts/nixosModules.holo-users.nix | 101 ++++++++++++++++-- modules/nixos/dev-minio.nix | 70 ++++++++++++ secrets/dev/secrets.yaml | 45 ++++++++ 6 files changed, 236 insertions(+), 14 deletions(-) create mode 100644 modules/nixos/dev-minio.nix create mode 100644 secrets/dev/secrets.yaml diff --git a/.sops.yaml b/.sops.yaml index 02f249d5..60a857c1 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -6,6 +6,7 @@ keys: - &steveej 6F7069FE6B96E894E60EC45C6EEFA706CB17E89B - &jost-s D299483493EAE6B2B3D892B6D33548FA55FF167F + - &dev age1fnmdutanvfsrhadap3qsmncjfa85x82qy8svy98ma4p37dglq45stcwk28 - &dweb-reverse-proxy age1ygzy9clj0xavlmau0ham7j5nw8yy4z0q8hvkfpdgwc4fcr8nufpqrdxgvx - &linux-builder-01 age1kxkr407jz77ljrhgsfwfmv2yvqjprc6unvx389xp2f48xj8r0vqq2wew5r - &x64-linux-dev-01 age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9 @@ -42,4 +43,10 @@ creation_rules: key_groups: - pgp: - *steveej - + - path_regex: ^secrets/dev/.+$ + key_groups: + - pgp: + - *steveej + age: + - *dev + - *x64-linux-dev-01 diff --git a/flake.lock b/flake.lock index b890665b..cbf6d8d2 100644 --- a/flake.lock +++ b/flake.lock @@ -867,11 +867,11 @@ ] }, "locked": { - "lastModified": 1715381426, - "narHash": "sha256-wPuqrAQGdv3ISs74nJfGb+Yprm23U/rFpcHFFNWgM94=", + "lastModified": 1717476296, + "narHash": "sha256-ScHe38Tr+TxGURC17kby4mIIxOG3aJvZWXzPM79UnEk=", "owner": "nix-community", "repo": "home-manager", - "rev": "ab5542e9dbd13d0100f8baae2bc2d68af901f4b4", + "rev": "095ef64aa3b2ab4a4f1bf07f29997e21e3a5576a", "type": "github" }, "original": { diff --git a/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix b/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix index 9304bd14..5b5ca42e 100644 --- a/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix +++ b/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix @@ -1,17 +1,19 @@ { - config, inputs, self, pkgs, lib, + config, ... -}: { +}: let +in { imports = [ inputs.disko.nixosModules.disko inputs.srvos.nixosModules.server inputs.srvos.nixosModules.hardware-hetzner-online-amd inputs.srvos.nixosModules.roles-nix-remote-builder self.nixosModules.holo-users + self.nixosModules.holo-users-interactive self.nixosModules.nix-build-distributor @@ -20,6 +22,19 @@ ../../nixos/shared.nix ../../nixos/shared-nix-settings.nix ../../nixos/shared-linux.nix + + { + home-manager.users.dev = {pkgs, ...}: { + home.packages = [ + # additional packages for this user go here + ]; + }; + } + + ]; + + nix.settings.system-features = [ + "big-parallel" ]; networking = { diff --git a/modules/flake-parts/nixosModules.holo-users.nix b/modules/flake-parts/nixosModules.holo-users.nix index c5164208..87aa94c5 100644 --- a/modules/flake-parts/nixosModules.holo-users.nix +++ b/modules/flake-parts/nixosModules.holo-users.nix @@ -1,15 +1,100 @@ { + self, inputs, lib, ... -}: { - flake.nixosModules.holo-users = { - users.users.root.openssh.authorizedKeys = { - keyFiles = - lib.attrValues - (lib.filterAttrs (name: _: lib.hasPrefix "keys_" name) inputs); - keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICHujII5RAwfEXNBYxKhWv2Wx/oHeHUTc8CACZ3M5W3p neonphog@gmail.com" +}: let + mkAuthorizedKeys = {...}: { + keyFiles = + lib.attrValues + (lib.filterAttrs (name: _: lib.hasPrefix "keys_" name) inputs); + keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICHujII5RAwfEXNBYxKhWv2Wx/oHeHUTc8CACZ3M5W3p neonphog@gmail.com" + ]; + }; +in { + flake.nixosModules.holo-users = {config, ...}: { + users.mutableUsers = false; + users.users.root.openssh.authorizedKeys = mkAuthorizedKeys {}; + }; + + flake.nixosModules.holo-users-interactive = {config, ...}: { + imports = [ + inputs.home-manager.nixosModules.home-manager + ]; + + # a generic dev user that can be used to have per-host home-manager environments for it. + # this adds no risk since all potential users already have access to the root account via their SSH credentials. + users.users.dev = { + home = "/home/dev"; + extraGroups = ["wheel"]; + openssh.authorizedKeys = mkAuthorizedKeys {}; + isNormalUser = true; + createHome = true; + }; + + sops.secrets.dev-age-key = { + sopsFile = self + "/secrets/dev/secrets.yaml"; + owner = "dev"; + }; + home-manager = { + useGlobalPkgs = true; + useUserPackages = true; + + sharedModules = [ + inputs.sops-nix.homeManagerModules.sops + ]; + users.dev = {pkgs, ...}: { + # Home Manager needs a bit of information about you and the + # paths it should manage. + home.username = "dev"; + home.homeDirectory = "/home/dev"; + + home.packages = [ + pkgs.coreutils + pkgs.neovim + ]; + + programs.bash.enable = true; + programs.bash.sessionVariables.SOPS_AGE_KEY_FILE = config.sops.secrets.dev-age-key.path; + + programs.direnv.enable = true; + # TODO: enable this once home-manager is bumped to >= release-24.05 + # programs.nix-direnv.enable = true; + + # This value determines the Home Manager release that your + # configuration is compatible with. This helps avoid breakage + # when a new Home Manager release introduces backwards + # incompatible changes. + # + # You can update Home Manager without changing this value. See + # the Home Manager release notes for a list of state version + # changes in each release. + home.stateVersion = "23.11"; + + # Let Home Manager install and manage itself. + programs.home-manager.enable = true; + + sops = { + age.keyFile = config.sops.secrets.dev-age-key.path; + defaultSopsFile = self + "/secrets/dev/secrets.yaml"; + }; + }; + }; + + security.sudo = { + enable = true; + execWheelOnly = true; + extraRules = [ + { + groups = ["wheel"]; + commands = [ + { + command = "ALL"; + options = ["NOPASSWD"]; + } + ]; + } ]; }; }; diff --git a/modules/nixos/dev-minio.nix b/modules/nixos/dev-minio.nix new file mode 100644 index 00000000..953f8d21 --- /dev/null +++ b/modules/nixos/dev-minio.nix @@ -0,0 +1,70 @@ +{ + config, + lib, + ... +}: let + name = "devMinio"; + cfg = config.services.${name}; +in { + options.services.${name} = { + enable = lib.mkEnableOption "dev minio server"; + rootDomain = lib.mkOption { + type = lib.types.str; + default = "dev.infra.holochain.org"; + }; + + s3Domain = lib.mkOption { + type = lib.types.str; + default = "s3.${cfg.rootDomain}"; + }; + + listenPort = lib.mkOption { + default = 9000; + type = lib.types.int; + }; + + consolePort = lib.mkOption { + default = 9001; + type = lib.types.int; + }; + + # TODO: revisit this as it's probably an anti-pattern + region = lib.mkOption { + description = "re-export of region"; + default = config.services.minio.region; + }; + }; + + config = { + sops.secrets.minio_root_credentials.sopsFile = ../../secrets/minio/server.yaml; + + services.minio = { + enable = true; + browser = true; + listenAddress = "127.0.0.1:${builtins.toString cfg.listenPort}"; + consoleAddress = "127.0.0.1:${builtins.toString cfg.consolePort}"; + rootCredentialsFile = config.sops.secrets.minio_root_credentials.path; + }; + + services.caddy.enable = true; + services.caddy.email = "admin@holochain.org"; + services.caddy.globalConfig = '' + auto_https disable_redirects + ''; + + services.caddy.virtualHosts."s3.${cfg.rootDomain}" = { + extraConfig = '' + reverse_proxy http://${config.services.minio.listenAddress} + ''; + }; + services.caddy.virtualHosts."s3-console.${cfg.rootDomain}" = { + extraConfig = '' + reverse_proxy http://${config.services.minio.consoleAddress} + ''; + }; + + networking.firewall.allowedTCPPorts = [ + 443 + ]; + }; +} diff --git a/secrets/dev/secrets.yaml b/secrets/dev/secrets.yaml new file mode 100644 index 00000000..22545760 --- /dev/null +++ b/secrets/dev/secrets.yaml @@ -0,0 +1,45 @@ +dev-age-key: ENC[AES256_GCM,data:VfeTQk/GLcriyJrgOtv1VrvJWAYjjXKV6LYOyiGuJgvsVcwTZlG4LGrDGGwG4etOmH+DAgCVP3CTJGtvjQi6XVQ42Xj/bVE9i/Hxe8ctC8dX2RFW0/zAv0McSZZTs9esFdgFVuBNjRmbFEUhnuKFr0EzXyOwFhsyGjwJAAWswRKfEuMfHBkaNkVjrmcF5GEo8N5kyKJUTk8DYDmu/ZwPmCoCmBag5CIVK4raIoIaJBHGez0WQ+J8+GPmIejQ,iv:N1Zhv1Y2BplIp4fqoxmNyWHxsupo4OJbQoEW1xWKNhU=,tag:9cCzy9dtisfsdmAt7qQnWQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1fnmdutanvfsrhadap3qsmncjfa85x82qy8svy98ma4p37dglq45stcwk28 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBMeGtuNHIvNzM2WDhlQWRH + bEU5ZTU3czFVNXBwOEF4MzVKM2FBbWg3VFdzCkJVZnNHWTlpRk53aEFEdVpJakY1 + aWU5NUp3azJSWWFtTmJIZURMQXBzMUkKLS0tIGV0WFl2dHNOTEFvNVhlWWVtcnUw + ckN4SjBSNFlMK2Qwd2doOVlXWVlOSXcKiIMRNcxUPG5ymE9SYEtdSI93KOSrwkUw + jud9IQVWrbDnkioj+iTcFfO0eWC9tlVpiXCCF1GRatqfjA4QXoJLSA== + -----END AGE ENCRYPTED FILE----- + - recipient: age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxUlA5RWUvTjZwelZ0ejRv + RFBNYWlRZ1dZQURMemxFMGRsb1haaStYVmgwCjNtV2JxK2FLcjVrSFBEc3krU2x2 + Yy9qQVhsLzBtUFJpSGo0VUFFVDc0dDAKLS0tIHpQd3ROeUx5QlRUN0hNRVZnZEtz + QVNvbU1PSSt3Nkt3WFRyTG1hY3JjRFEKelkS9FVL/gW1BbfND6oWFJOsJ03Rqei3 + skxzskCZGl+P0/mfPvBS8cnAOTQiWWLe3e0rG/uJTk4o2CeLpIO6fw== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-06-26T15:49:56Z" + mac: ENC[AES256_GCM,data:zCPSZx6UjI3HHYy7l2imlim9DcPuH461/0dvxfuKlwXynxowa6PQ8yWNDryd4P3lMjQikcau1o/90fn31ptiFfDEz+Uz7tlcg6zHXiMYMDy2VLoRtEH46TvZ24WUocLB3V8L3kWdfnLdX6+UKBDFGke5AruIFbGM4xbh+TRFrp8=,iv:5bYW0No97RJ34f9D02jw9FjJbuue0SMpmIYY6CtmuZU=,tag:q34U5JcGmJWLpEeUyr5ISQ==,type:str] + pgp: + - created_at: "2024-06-20T17:55:23Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQEMA0SHG/zF3227AQf/X3wVZlUg/hMiSAHk9hm5cfKp0bVKRecLR4726QVKK5NN + OZPmddpHfbUjnj3i2z8BY2a8GHtek1zVMaAKPdCarwu5iF3rPZtjajcjgxDRiBwG + tjVuY/MZ9jGR4ah2z7mdjKGw39YVb22uCUt281Jn0faGbMaJRYF4vO3X000eIszc + 8hjNj293gi8Cz0oBMU85PfcDRFoSyQ1qyw0x4sj/qM16whGjU7S8Jm3GQ4Kn80DK + y7eLzfEgnmeO77twNChm5cKPikv6H1HROkEnGgeFV165VWUAtS87+KuOENpOnNgi + x9sVa7Gz0LJx9XXf9l6Tee5JBE66oe2yHq+ZCPHJYdJeAeqS429Du9+NvwwA1CwL + 3v+Ohox739RVq6a0ARyHeLGknP9NjDI6V1DqizjbhMBNXgvW/HKeC760DBpYviNB + /fP/WvTRfbvmAoJ7BWkLHM5qQFd2cfP6lW9vtOms7A== + =iWez + -----END PGP MESSAGE----- + fp: 6F7069FE6B96E894E60EC45C6EEFA706CB17E89B + unencrypted_suffix: _unencrypted + version: 3.8.1 From 61d2fc281afbfad4a45b1b6f633649318492118f Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Wed, 26 Jun 2024 17:57:29 +0200 Subject: [PATCH 2/4] feat: minio S3 on x64-linux-dev-01 for flist storage, adapt scripts & pulumi * publicly reachable minio S3 with TLS reverse-proxy * secrets for pushing objects as "dev" * zos- scripts push to minio by default * adapt pulumi to pull the default flist from here --- .sops.yaml | 13 +++++ flake.nix | 38 +++++++++++--- .../configuration.nix | 5 +- .../configuration.nix | 4 ++ modules/flake-parts/packages.zos-utils.nix | 50 +++++++++---------- pulumi/main.go | 2 +- secrets/linux-builder-02/secrets.yaml | 21 -------- secrets/minio/server.yaml | 45 +++++++++++++++++ secrets/x64-linux-dev-01/secrets.yaml | 38 ++++++++++++++ 9 files changed, 157 insertions(+), 59 deletions(-) delete mode 100644 secrets/linux-builder-02/secrets.yaml create mode 100644 secrets/minio/server.yaml create mode 100644 secrets/x64-linux-dev-01/secrets.yaml diff --git a/.sops.yaml b/.sops.yaml index 60a857c1..8bb3dac7 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -27,6 +27,12 @@ creation_rules: key_groups: - age: - *linux-builder-01 + - path_regex: ^secrets/x64-linux-dev-01/[^/]+$ + key_groups: + - pgp: + - *steveej + age: + - *x64-linux-dev-01 - path_regex: ^secrets/nomad/.+$ key_groups: - pgp: @@ -50,3 +56,10 @@ creation_rules: age: - *dev - *x64-linux-dev-01 + - path_regex: ^secrets/minio/.+$ + key_groups: + - pgp: + - *steveej + age: + - *dev + - *x64-linux-dev-01 diff --git a/flake.nix b/flake.nix index 5bad8c23..f176c073 100644 --- a/flake.nix +++ b/flake.nix @@ -188,6 +188,7 @@ pkgs.caddy inputs'.threefold-rfs.packages.default + pkgs.minio-client pkgs.jq pkgsPulumi.pulumictl @@ -212,14 +213,35 @@ NOMAD_CACERT = nomadCaCert; NOMAD_CLIENT_CERT = nomadClientCert; - shellHook = '' - set -x - REPO_SECRETS_DIR="''${HOME:?}/.holochain-infra-secrets" - mkdir -p ''${REPO_SECRETS_DIR} - chmod 700 ''${REPO_SECRETS_DIR} - export NOMAD_CLIENT_KEY="''${REPO_SECRETS_DIR}/global-cli-nomad-key"; - sops -d secrets/nomad/cli/keys.yaml | yq '.global-cli-nomad-key' > ''${NOMAD_CLIENT_KEY:?} - ''; + shellHook = let + devMinioOsConfig = self.nixosConfigurations.x64-linux-dev-01.config; + in + '' + if sops -d secrets/nomad/cli/keys.yaml 2>&1 >/dev/null; then + REPO_SECRETS_DIR="''${HOME:?}/.holochain-infra-secrets" + mkdir -p ''${REPO_SECRETS_DIR} + chmod 700 ''${REPO_SECRETS_DIR} + export NOMAD_CLIENT_KEY="''${REPO_SECRETS_DIR}/global-cli-nomad-key"; + sops -d secrets/nomad/cli/keys.yaml | yq '.global-cli-nomad-key' > ''${NOMAD_CLIENT_KEY:?} + fi + '' + + (let + minioUserPass = ''''${MINIO_ROOT_USER}:''${MINIO_ROOT_PASSWORD}''; + minioDevHost = devMinioOsConfig.services.devMinio.s3Domain + ":443"; + minioDevLocalHost = "127.0.0.1:${builtins.toString devMinioOsConfig.services.devMinio.listenPort}"; + minioRegion = devMinioOsConfig.services.devMinio.region; + in '' + if sops -d secrets/minio/server.yaml 2>&1 >/dev/null; then + source <(sops -d secrets/minio/server.yaml | yq '.minio_root_credentials') + + export MC_HOST_devminio_local="http://${minioUserPass}@${minioDevLocalHost}"; + export MC_HOST_devminio="https://${minioUserPass}@${minioDevHost}" + + export RFS_HOST_devminio_region="${minioRegion}" + export RFS_HOST_devminio_local="s3://${minioUserPass}@${minioDevLocalHost}" + export RFS_HOST_devminio="s3s://${minioUserPass}@${minioDevHost}" + fi + ''); }; packages = diff --git a/modules/flake-parts/nixosConfigurations.dweb-reverse-tls-proxy/configuration.nix b/modules/flake-parts/nixosConfigurations.dweb-reverse-tls-proxy/configuration.nix index 259cb45d..597e859b 100644 --- a/modules/flake-parts/nixosConfigurations.dweb-reverse-tls-proxy/configuration.nix +++ b/modules/flake-parts/nixosConfigurations.dweb-reverse-tls-proxy/configuration.nix @@ -183,8 +183,9 @@ in { hackathon.events.${fqdn2domain}. A 10.1.3.37 amsterdam2023.events.${fqdn2domain}. A 10.1.3.187 - sj-bm-hostkey0.dev.${fqdn2domain}. A 185.130.224.33 - x64-linux-dev-01.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName} + x64-linux-dev-01.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName} + s3.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName} + s3-console.dev.${fqdn2domain}. A ${self.nixosConfigurations.x64-linux-dev-01.config.hostName} turn-0.${fqdn2domain}. A ${self.nixosConfigurations.turn-0.config.services.holochain-turn-server.address} signal-0.${fqdn2domain}. A ${self.nixosConfigurations.turn-0.config.services.tx5-signal-server.address} diff --git a/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix b/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix index 5b5ca42e..ed4267fa 100644 --- a/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix +++ b/modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix @@ -31,6 +31,10 @@ in { }; } + ../../nixos/dev-minio.nix + { + services.devMinio.enable = true; + } ]; nix.settings.system-features = [ diff --git a/modules/flake-parts/packages.zos-utils.nix b/modules/flake-parts/packages.zos-utils.nix index 81b12d7a..0214b3f8 100644 --- a/modules/flake-parts/packages.zos-utils.nix +++ b/modules/flake-parts/packages.zos-utils.nix @@ -31,29 +31,16 @@ ''; }; - # TODO: automate proper minio hosting. this is exemplary only and requires imperative setup of minio - zos-vm-serve-s3 = pkgs.writeShellApplication { - name = "zos-vm-serve-s3"; - runtimeInputs = [ - pkgs.minio - ]; - text = '' - set -ueE -o pipefail - - cd .minio - - env \ - MINIO_ROOT_USER=minioadmin \ - MINIO_ROOT_PASSWORD="$(cat minioadmin.key)" \ - minio server --console-address ":9001" storage - ''; - }; - zos-vm-publish-s3 = let - s3BaseUrl = "sj-bm-hostkey0.dev.infra.holochain.org"; - s3ListenUrl = "${s3BaseUrl}:9000"; - s3HttpUrl = "https://${s3BaseUrl}/s3"; + # TODO: document these: explain the relationship to the variables in the devShell's shellHook; if viable give them a common source of truth + s3BaseUrl = "dev.infra.holochain.org"; + s3HttpUrl = "https://s3.${s3BaseUrl}"; + + # TODO: programmatically ensure this exists s3Bucket = "tfgrid-eval"; + + # TODO: this is faster however restricts publishing to the server itself + s3Alias = "devminio_local"; in pkgs.writeShellApplication { name = "zos-vm-publish-s3"; @@ -73,12 +60,16 @@ mkdir -p "$workDir" cd "$workDir" + s3_remote="''${MC_HOST_devminio_local:?}" + s3_remote="''${s3_remote/http:\/\//}" + # mc rm --recursive --force localhost/${s3Bucket} || echo removal failed env RUST_MIN_STACK=8388608 \ - rfs pack -m result.fl -s s3://minioadmin:"$(cat ../../.minio/minioadmin.key)"@${s3ListenUrl}/${s3Bucket}\?region=us-east-1 "$rootfs/" | tee rfs-pack.log + rfs pack -m result.fl -s "''${RFS_HOST_devminio:?}/${s3Bucket}?region=''${RFS_HOST_devminio_region:?}" "$rootfs/" | tee rfs-pack.log + + mc cp result.fl ${s3Alias}/${s3Bucket}/"$rootfsBase".fl - # TODO: document or automate setting up the alias "localhost" - mc cp result.fl localhost/${s3Bucket}/"$rootfsBase".fl + # the final URL doesn't have the bucket name as it's implied as the default bucket. echo ${s3HttpUrl}/${s3Bucket}/"$rootfsBase".fl > public-url touch published @@ -208,9 +199,12 @@ kernel="$rootfs/boot/vmlinuz" initram="$rootfs/boot/initrd.img" + # FIXME: can't handle path ending in '/' workDir="$rootfs.work" mountDir="$workDir/mnt" mkdir -p "$mountDir" + # set it to read-only by default. the mount will be writable. + chmod 440 "$mountDir" socket="$workDir/virtiofs.sock" @@ -219,7 +213,9 @@ exit 1 } - rfs mount -m "$workDir"/result.fl "$mountDir" > "$workDir"/rfs_mount.log 2>&1 & + # FIXME: check whether the mount was successful + # FIXME: don't rely on sudo + sudo rfs mount -m "$workDir"/result.fl "$mountDir" 2>&1 | tee "$workDir"/rfs_mount.log & mountpid="$!" sleep 3 @@ -243,8 +239,8 @@ sudo kill "$fspid" rm -rf "$socket" - kill "$mountpid" - umount --lazy "$mountDir" + sudo kill "$mountpid" + sudo umount --lazy "$mountDir" rmdir "$mountDir" ) } diff --git a/pulumi/main.go b/pulumi/main.go index ef30d190..d1b1e1f2 100644 --- a/pulumi/main.go +++ b/pulumi/main.go @@ -81,7 +81,7 @@ func main() { "SSH_KEY": pulumi.String("ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAIODJoJ7Chi8jPTGmKQ5MlB7+TgNGznreeRW/K34v1ey23/FlnIxP9XyyLkzojKALTfAQYgqzrQV3HDSRwhd1rXB7YLq1/CiVWRJvDMTkJiOCV515eiUJGXu1G8e12d/USPNBMEzMJGvqBCIGYen5OxXkyIHIREfePNi5k337G5z9fiuiggxJl9ty6qZ4XIRgFQj9jAoShixP/+99I7XrGWeFQ1BmLZWzi20SQGKvogYnOszDZFqBAHGFnCFYHaTz2jOXXCtQsa27gr8D2iLRFaxvhB7XMK+VbpDcZGjmfRJ701XxFv15GFnFAV71hTaYqj/Ebpw9Vs02+gUp3+tt cardno:17_673_080"), }, - Flist: pulumi.String("https://sj-bm-hostkey0.dev.infra.holochain.org/s3/tfgrid-eval/tfgrid-base.20240517.114850.fl"), + Flist: pulumi.String("https://s3.dev.infra.holochain.org/tfgrid-eval/tfgrid-base.20240624.183001.fl"), Memory: pulumi.Int(512), Name: pulumi.String("tfgrid_base"), Network_name: grid_network.Name, diff --git a/secrets/linux-builder-02/secrets.yaml b/secrets/linux-builder-02/secrets.yaml deleted file mode 100644 index 27b11916..00000000 --- a/secrets/linux-builder-02/secrets.yaml +++ /dev/null @@ -1,21 +0,0 @@ -gh_hra2_pat4: ENC[AES256_GCM,data:RhbPMa7JpZtxZzIWKBYdLodwYZhCupWmzZcIT/+1OUYFyMlFxNXWvA==,iv:XEZmlAtO+jLZlTXlSmLvi1VVUbLYe1kVmqw8ylCq7d8=,tag:EkggPJa3Du1frkxzl/8A7g==,type:str] -sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] - age: - - recipient: age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwVHQvbGwwaXkvdk15WW90 - VVUvZDdKUGZIM3lFUERiWVUraTVHUm8vWHhzClM2WU5VK3BVU2dlanlaWHlkQitn - N0lJN2FFOU1pdHFhcGQ4SnBtZWZFcEEKLS0tIHM1eWc0Wnp2YzBBRDhpbUY4SDB5 - ZGJuKy9CVGkyWW14YlJDWXRZcnZWQXcKof00tuXDus26+8xbKjyzkvY5JcoAxHZZ - X0lVNQW1DU5/Jy5NRshnBlRm5WifnwE0SbjF2lpTSmgOSBCYByMTsg== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-06-06T09:07:57Z" - mac: ENC[AES256_GCM,data:/ILjd3srjGGxw42P6HMJvlQJK2QlNgnJpbxhlPIahDzSfhGSJcPpl6BSuU5ETCaIsALuyFARtRKJe3eBYYLAVb9hv7rjPvhkbNCviSgfKwM4HUMuxyk7g88OPE4T6rANswFwz6IuZddAo0TtnAZYHoy7CfHWGubi9VnlSNC88PQ=,iv:t1yz+yVFleNbmIvJ2Vn8vQWoick7qrV2fmvdeT/UaZ0=,tag:xwKWS51eFZ4j3WVuz4a/Pg==,type:str] - pgp: [] - unencrypted_suffix: _unencrypted - version: 3.8.1 diff --git a/secrets/minio/server.yaml b/secrets/minio/server.yaml new file mode 100644 index 00000000..78e29b6a --- /dev/null +++ b/secrets/minio/server.yaml @@ -0,0 +1,45 @@ +minio_root_credentials: ENC[AES256_GCM,data:8CvFLwseK8nYwkyImcjkTL+L96PllC4aFdG/gorcV/paqA/221ugFT+WMLxlDMAwGQ4SqentncfqouhTT2Rig7eA8MbgNm7xwdzfu7erRd8a9bkc4O7s0WbL+SABdPB9X0Q=,iv:oPtfZ8dGm4yDseFd/BuLZ3s7nqnSQIs0b9ZpGrYL9iI=,tag:YBEsI0D/MO4rH52CHRGYJw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1fnmdutanvfsrhadap3qsmncjfa85x82qy8svy98ma4p37dglq45stcwk28 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzL0YxY1hkaU01VnpXR0pz + OGN0VGc3QlJTVU9sdUwvcjNDYklrM0dhRmpRCmZsQjNoYm5HMHVlNi9ZcElMRFQ2 + ZTNTR2J0Y0lRdCtaN3lydS9jY2lSOW8KLS0tIEY0a2d3Z2hNZm9yd3VsdjY3ZVhE + aGpTeUR6S0J1bTdlMXJZWDFHTTdFOUUKW3I1lCLAxBjubEdnG6m4kxChTGfX4MjU + VJ+GTdz6Iv6exV4TONQFO/Xety7+W/DlAQch3ulhHoiNObKnIf5Q/A== + -----END AGE ENCRYPTED FILE----- + - recipient: age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0SWp0c2ZtYTlHVHRmS000 + SWdoUndUYWNRWVpOR2NlY0dVRi8vNWtMa0JBCjNJaWpjYklROEh5aVdNRENDRVlU + dWNJMEpmQXgwT1o4Q3RJbWw4NEhLTVUKLS0tIHJubW9HelY3ZzNYb1VYeGsvQUdm + MXdjM1IrcTliT0o2TUdJTnRKVzEya00K5IsrInmrFXP+G/KqGJyI+q056uO+HDDz + 238xZvJUonkTvUFavQaPWhBjpayKu06b1xHY4Xr/TZkoWs9B5pSENQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-06-25T09:20:05Z" + mac: ENC[AES256_GCM,data:v/OoZ6snh0ZpWYQCU1c+WBoMJoJiNGiw4n/45wnYEglaVIaiNDoGWL/kx6JFBAkFu7EAYqV3y/jGRi0rmN09kLBoDIpPrGCgG345Mh3l4ffJjPiZEtAtzYqb/x9DxefNCA1HJphK4ZKoOk9kVH6c7GVmY1+2r6csRznejlbldC4=,iv:QW53UQBARGVKLtzT14OmKXdlxH4l0lpfCXcQCDFyusI=,tag:VdRSi9/Mr5NbCuJQm3L4oQ==,type:str] + pgp: + - created_at: "2024-06-25T12:34:07Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQEMA0SHG/zF3227AQf/adv0266KfOkuMv27DpYih1Yxgd4SuVC1yEI7CY/6K+uX + gqQ1pS6BOnJobBD5VnEP/Clvpf+vDCsy/fN22kKieAAdW9kp1yUdi6Hogd0GzbPj + IDZ89urOuWvLa9i8P18Nw2bHbrvsL4nae4UQEWc/PS3+jlDNK/4FX2xBJ/pMzKd+ + QPumYyZpEi0mXDxjVZsJB4f/EagBA7ucODLB8jxTHEll+lWifBCq5X7SpK78uBwq + WZXec/6APHW+0tJ6yBZQs/L2Kve7L5uAAjmLqNC7o2kI78Qs4TOX9ERHoHuFZxKi + ghrV8+b+SbF2rdX7AS5DdIY/doi3YU0ftwV+Ku5dtdJcARVHrsiuzmnMQJOsuxqY + lglNlzfDWCpvQi8wC4lr3mSg/zmt9rSxuZTpMjeFG1aTTFJljguovfHbDWbGaqWY + sJdaDTEtxHGdN2ZGzn/ubdzcAKi+xQWi4R03MiI= + =AdR8 + -----END PGP MESSAGE----- + fp: 6F7069FE6B96E894E60EC45C6EEFA706CB17E89B + unencrypted_suffix: _unencrypted + version: 3.8.1 diff --git a/secrets/x64-linux-dev-01/secrets.yaml b/secrets/x64-linux-dev-01/secrets.yaml new file mode 100644 index 00000000..3c345595 --- /dev/null +++ b/secrets/x64-linux-dev-01/secrets.yaml @@ -0,0 +1,38 @@ +GARAGE_ADMIN_TOKEN: ENC[AES256_GCM,data:jSPFrFJisaQVgFbW+SWt+zJz6f0RKk5iktgtawm3GJ74k2APptUj1PLpaH4=,iv:Kd5UIkjfhSMksC0GYSK2/V3aiviUbNBqvc6C9xLBgnY=,tag:ZB4iHhHgoo6fll/uCtIe9w==,type:str] +GARAGE_METRICS_TOKEN: ENC[AES256_GCM,data:D+6/R2xLWmKGI35xHSVsx16IrPZrWmHl/x6Sc9j5+VuvnY871HiJqVvkfwM=,iv:9G/InGhhhAe8TSEHP8yPqr0xaZiZfy6lqoCyQ5hV9YU=,tag:cKgyEGiRvll9/D6hmfzQ4g==,type:str] +GARAGE_RPC_SECRET: ENC[AES256_GCM,data:5GjuiZwGRqxGaXAT/m2Gh6SzEXevrkfrrvA+9Dyyd1FPibmEcjoYrLKvjFvqJsRod/xnM6T+qdGsqx/OkG8bhw==,iv:EDjtXgFGVQn73UajlyfJckm1JFaNxsNGzPtJD76zCF4=,tag:mSlOHYqMVBCq31niBaCtIQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByNkM4cUJPQ0RrZ3hoZmt3 + ZEplM0J2YXpRUkVSeUFMQi9QeHNNVW9qRWxZClpESlNvZjlZejYxSWFqQVcybTI0 + KzhTNzF3QnpseDVrN0Q5QStNVit1MzQKLS0tIFRNR0lja3pSdjlXVGZLRW1JdjJt + VlpvVWVLMmU3bVc3VGZweWFqZGMwUmMKW6LxpwS7O6xpkOApIbZ9/EHixGn0KNMQ + IqslGdqGL8PI5lubED+VmwWERsXMi4Ywqx/U8fc4FTbSXu2nQXOqGA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-06-20T17:13:39Z" + mac: ENC[AES256_GCM,data:BatqYuU+5VLDgizNYqdKecYDl5K6O5b9oySC3sSOHcPPAsJ8QvOfJ9uGmTzvyUWUIiKCCCZf2aSKac/3cYbn5i0hvt8AfRbLReERPT32US/5dpGAl3Cy5iGUoSxqv1H+LJZO5yzXjaRaPhrHIQWYMcNp5IA+/KD2YWJIbDqAMBM=,iv:aEI0FG7FhmjmuabqZYlGQ+2l4/EwM/Kryrewl/1Htjw=,tag:9pLrM97uEQHiezsvt/cakQ==,type:str] + pgp: + - created_at: "2024-06-20T10:37:35Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + hQEMA0SHG/zF3227AQgAloQvBhdn44VFM4jpHdGrRVqMaD2zfh80XR/WXdDq6yFr + 6/QhB8wYuMpW/RHd99wQECDXb18vETgww+adrM1l7t1HqHfF6JmKrip8UGZo73Xk + 7TPIDQvZrJCyTekQlO9Q2E9EVi+v48uzAjj3z+A29kKBCuQqDnVbHOG4QfEVVZWj + B2DLJ3RbEUOWOGgMqe/J9+5Nn+633ToCQ52pcNP0lBDDWxg5dtsd+c6zL8hpnr09 + wYeGEDfuxbNuh1bOBesaX/vjfAe/KLZhaIHDkJ+ot/Eh0WTlDIbuNU3Hg3kcqGDR + hqVMb5SIkLWN719nRrd5sm8jYV2p9ED6K6JP024xDtJcAcLVwxOZ1bkmPJ4pZPB1 + WS5H/9DANH8Ka7mZ94Mxu/gJf/lXj+IMKmo+Lz/M79IgiBc/H75Sul4Y6jgHtNbu + ljxdJ41ALZyioJrZbOlF9t/OaDKIiphQM4lgG2E= + =lpKT + -----END PGP MESSAGE----- + fp: 6F7069FE6B96E894E60EC45C6EEFA706CB17E89B + unencrypted_suffix: _unencrypted + version: 3.8.1 From b5e8c12b44a7b7f5d34de9cfaa9ad3cc8f8d6efe Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Wed, 26 Jun 2024 18:04:25 +0200 Subject: [PATCH 3/4] chore: update pulumi stack metadata --- .../.pulumi/stacks/holochain-infra/dev.json | 2 +- .../stacks/holochain-infra/dev.json.attrs | 2 +- .../stacks/holochain-infra/dev.json.bak | 39 +------------------ .../stacks/holochain-infra/dev.json.bak.attrs | 2 +- 4 files changed, 5 insertions(+), 40 deletions(-) diff --git a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json index 5366664a..6fbda842 100644 --- a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json +++ b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json @@ -4,7 +4,7 @@ "stack": "organization/holochain-infra/dev", "latest": { "manifest": { - "time": "2024-06-18T21:43:06.192982843+02:00", + "time": "2024-06-26T17:47:05.154786843+02:00", "magic": "6cece4896dda855f1f8eeb278295600f55147c3a4c822170dd54c64d426a45f0", "version": "v3.116.0" }, diff --git a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.attrs b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.attrs index f5917752..4a8c71e0 100644 --- a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.attrs +++ b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.attrs @@ -1 +1 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"67cPV+aK88FxD+NCXt+73Q=="} +{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"QBv1jrqEVzsJvsx0QknZLQ=="} diff --git a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak index 96f2773f..439f5bd1 100644 --- a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak +++ b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak @@ -4,7 +4,7 @@ "stack": "organization/holochain-infra/dev", "latest": { "manifest": { - "time": "2024-06-18T21:43:06.190613726+02:00", + "time": "2024-06-26T12:45:36.244633923+02:00", "magic": "6cece4896dda855f1f8eeb278295600f55147c3a4c822170dd54c64d426a45f0", "version": "v3.116.0" }, @@ -13,42 +13,7 @@ "state": { "salt": "v1:Pqc+zMVi/jk=:v1:qrWBTYDIcH2WqUmg:1THU1HTwRjJH4PGbQ3nmjuSlU+amrg==" } - }, - "resources": [ - { - "urn": "urn:pulumi:dev::holochain-infra::pulumi:providers:random::default_4_16_2", - "custom": true, - "id": "9891b164-4d3d-42c9-8fba-a1562edb14b2", - "type": "pulumi:providers:random", - "inputs": { - "version": "4.16.2" - }, - "outputs": { - "version": "4.16.2" - }, - "created": "2024-06-18T19:25:20.331328844Z", - "modified": "2024-06-18T19:25:20.331328844Z" - } - ], - "pending_operations": [ - { - "resource": { - "urn": "urn:pulumi:dev::holochain-infra::pulumi:providers:random::default_4_16_2", - "custom": true, - "id": "9891b164-4d3d-42c9-8fba-a1562edb14b2", - "type": "pulumi:providers:random", - "inputs": { - "version": "4.16.2" - }, - "outputs": { - "version": "4.16.2" - }, - "created": "2024-06-18T19:25:20.331328844Z", - "modified": "2024-06-18T19:25:20.331328844Z" - }, - "type": "deleting" - } - ] + } } } } diff --git a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak.attrs b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak.attrs index 65f3b5ef..42346484 100644 --- a/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak.attrs +++ b/pulumi-state/.pulumi/stacks/holochain-infra/dev.json.bak.attrs @@ -1 +1 @@ -{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"Ts2Gcm66tXVyriUumxp7FQ=="} +{"user.cache_control":"","user.content_disposition":"","user.content_encoding":"","user.content_language":"","user.content_type":"text/plain; charset=utf-8","user.metadata":null,"md5":"psvrAQRHecZCHN9iWMVQnQ=="} From ebd5cf2bce503ccd345c66efa0024b76440591d7 Mon Sep 17 00:00:00 2001 From: Stefan Junker Date: Wed, 26 Jun 2024 18:04:47 +0200 Subject: [PATCH 4/4] feat: add dev-garage module for future reference intermediately i used garage as an S3 storage. it didn't fit the workflow well because garage doesn't currently support anonymous access to S3 objects. `rfs pack` stores the s3 credentials that it uses for pushing to the store into the resulting flist (by default with a stripped password). options for making this work are * adding anonymous download support to garage * creating a read-only credential and either modify rfs to store alternative credentials or post-process the flist (sqlite3 db). --- modules/nixos/dev-garage.nix | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 modules/nixos/dev-garage.nix diff --git a/modules/nixos/dev-garage.nix b/modules/nixos/dev-garage.nix new file mode 100644 index 00000000..5ca10656 --- /dev/null +++ b/modules/nixos/dev-garage.nix @@ -0,0 +1,92 @@ +{ + self, + config, + ... +}: let + root_domain = "dev.infra.holochain.org"; + s3_web_port = "3902"; + s3_port = "3900"; +in { + users.groups.garage-secrets.members = [ + "dev" + ]; + + sops = { + defaultSopsFile = self + "/secrets/${config.networking.hostName}/secrets.yaml"; + secrets = { + GARAGE_ADMIN_TOKEN = { + group = "garage-secrets"; + mode = "440"; + }; + GARAGE_METRICS_TOKEN = { + group = "garage-secrets"; + mode = "440"; + }; + GARAGE_RPC_SECRET = { + group = "garage-secrets"; + mode = "440"; + }; + }; + }; + + systemd.services.garage.serviceConfig.Group = "garage-secrets"; + /* + post deployment actions taken to get the node ready for storing files + + ``` + garage status + garage layout assign fdf468cca3934a18 -c 100G -z dc0 + garage layout apply --version 1 + ``` + */ + services.garage = { + enable = true; + package = self.inputs.nixpkgs-24-05.legacyPackages.${pkgs.stdenv.system}.garage_1_0_0; + settings = { + # it's *NOT* world-readable, however not was garage exepects either + # Jun 20 17:27:39 x64-linux-dev-01 garage[1701365]: Error: File /run/secrets/GARAGE_RPC_SECRET is world-readable! (mode: 0100440, expected 0600) + allow_world_readable_secrets = true; + + rpc_bind_addr = "[::]:3901"; + rpc_secret_file = config.sops.secrets.GARAGE_RPC_SECRET.path; + + s3_api = { + api_bind_addr = "[::]:${s3_port}"; + s3_region = "garage"; + root_domain = ".s3.${root_domain}"; + }; + + s3_web = { + bind_addr = "[::]:${s3_web_port}"; + root_domain = ".web.${root_domain}"; + }; + admin = { + api_bind_addr = "0.0.0.0:3903"; + metrics_token_file = config.sops.secrets.GARAGE_METRICS_TOKEN.path; + admin_token_file = config.sops.secrets.GARAGE_ADMIN_TOKEN.path; + }; + }; + }; + + services.caddy.enable = true; + services.caddy.email = "admin@holochain.org"; + services.caddy.globalConfig = '' + auto_https disable_redirects + ''; + + services.caddy.virtualHosts."s3web.${root_domain}" = { + extraConfig = '' + reverse_proxy http://127.0.0.1:${s3_web_port} + ''; + }; + services.caddy.virtualHosts."s3.${root_domain}" = { + extraConfig = '' + reverse_proxy http://127.0.0.1:${s3_port} + ''; + }; + + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; +}