Merge pull request #131 from holochain/s3-storage-for-flists-minio

feat: set up x64-linux-dev-01 as a (tfgrid flists) development server, including a persistent s3 storage for flist hosting

.sops.yaml

@@ -6,6 +6,7 @@
 keys:
   - &steveej 6F7069FE6B96E894E60EC45C6EEFA706CB17E89B
   - &jost-s D299483493EAE6B2B3D892B6D33548FA55FF167F
+  - &dev age1fnmdutanvfsrhadap3qsmncjfa85x82qy8svy98ma4p37dglq45stcwk28
   - &dweb-reverse-proxy age1ygzy9clj0xavlmau0ham7j5nw8yy4z0q8hvkfpdgwc4fcr8nufpqrdxgvx
   - &linux-builder-01 age1kxkr407jz77ljrhgsfwfmv2yvqjprc6unvx389xp2f48xj8r0vqq2wew5r
   - &x64-linux-dev-01 age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9
@@ -26,6 +27,12 @@ creation_rules:
     key_groups:
     - age:
       - *linux-builder-01
+  - path_regex: ^secrets/x64-linux-dev-01/[^/]+$
+    key_groups:
+    - pgp:
+      - *steveej
+      age:
+      - *x64-linux-dev-01
   - path_regex: ^secrets/nomad/.+$
     key_groups:
     - pgp:
@@ -42,4 +49,17 @@ creation_rules:
     key_groups:
     - pgp:
       - *steveej
-
+  - path_regex: ^secrets/dev/.+$
+    key_groups:
+    - pgp:
+      - *steveej
+      age:
+      - *dev
+      - *x64-linux-dev-01
+  - path_regex: ^secrets/minio/.+$
+    key_groups:
+    - pgp:
+      - *steveej
+      age:
+      - *dev
+      - *x64-linux-dev-01

flake.nix

@@ -188,6 +188,7 @@
                 pkgs.caddy
 
                 inputs'.threefold-rfs.packages.default
+                pkgs.minio-client
 
                 pkgs.jq
                 pkgsPulumi.pulumictl
@@ -212,14 +213,35 @@
             NOMAD_CACERT = nomadCaCert;
             NOMAD_CLIENT_CERT = nomadClientCert;
 
-            shellHook = ''
-              set -x
-              REPO_SECRETS_DIR="''${HOME:?}/.holochain-infra-secrets"
-              mkdir -p ''${REPO_SECRETS_DIR}
-              chmod 700 ''${REPO_SECRETS_DIR}
-              export NOMAD_CLIENT_KEY="''${REPO_SECRETS_DIR}/global-cli-nomad-key";
-              sops -d secrets/nomad/cli/keys.yaml | yq '.global-cli-nomad-key' > ''${NOMAD_CLIENT_KEY:?}
-            '';
+            shellHook = let
+              devMinioOsConfig = self.nixosConfigurations.x64-linux-dev-01.config;
+            in
+              ''
+                if sops -d secrets/nomad/cli/keys.yaml 2>&1 >/dev/null; then
+                  REPO_SECRETS_DIR="''${HOME:?}/.holochain-infra-secrets"
+                  mkdir -p ''${REPO_SECRETS_DIR}
+                  chmod 700 ''${REPO_SECRETS_DIR}
+                  export NOMAD_CLIENT_KEY="''${REPO_SECRETS_DIR}/global-cli-nomad-key";
+                  sops -d secrets/nomad/cli/keys.yaml | yq '.global-cli-nomad-key' > ''${NOMAD_CLIENT_KEY:?}
+                fi
+              ''
+              + (let
+                minioUserPass = ''''${MINIO_ROOT_USER}:''${MINIO_ROOT_PASSWORD}'';
+                minioDevHost = devMinioOsConfig.services.devMinio.s3Domain + ":443";
+                minioDevLocalHost = "127.0.0.1:${builtins.toString devMinioOsConfig.services.devMinio.listenPort}";
+                minioRegion = devMinioOsConfig.services.devMinio.region;
+              in ''
+                if sops -d secrets/minio/server.yaml 2>&1 >/dev/null; then
+                  source <(sops -d secrets/minio/server.yaml | yq '.minio_root_credentials')
+
+                  export MC_HOST_devminio_local="http://${minioUserPass}@${minioDevLocalHost}";
+                  export MC_HOST_devminio="https://${minioUserPass}@${minioDevHost}"
+
+                  export RFS_HOST_devminio_region="${minioRegion}"
+                  export RFS_HOST_devminio_local="s3://${minioUserPass}@${minioDevLocalHost}"
+                  export RFS_HOST_devminio="s3s://${minioUserPass}@${minioDevHost}"
+                fi
+              '');
           };
 
         packages =

modules/flake-parts/nixosConfigurations.dweb-reverse-tls-proxy/configuration.nix

@@ -183,8 +183,9 @@ in {
       hackathon.events.${fqdn2domain}.         A       10.1.3.37
       amsterdam2023.events.${fqdn2domain}.     A       10.1.3.187
 
-      sj-bm-hostkey0.dev.${fqdn2domain}.       A       185.130.224.33
-      x64-linux-dev-01.dev.${fqdn2domain}.    A       ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
+      x64-linux-dev-01.dev.${fqdn2domain}.       A       ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
+      s3.dev.${fqdn2domain}.                     A       ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
+      s3-console.dev.${fqdn2domain}.             A       ${self.nixosConfigurations.x64-linux-dev-01.config.hostName}
 
       turn-0.${fqdn2domain}.                     A       ${self.nixosConfigurations.turn-0.config.services.holochain-turn-server.address}
       signal-0.${fqdn2domain}.                   A       ${self.nixosConfigurations.turn-0.config.services.tx5-signal-server.address}

modules/flake-parts/nixosConfigurations.x64-linux-dev-01/configuration.nix

@@ -1,17 +1,19 @@
 {
-  config,
   inputs,
   self,
   pkgs,
   lib,
+  config,
   ...
-}: {
+}: let
+in {
   imports = [
     inputs.disko.nixosModules.disko
     inputs.srvos.nixosModules.server
     inputs.srvos.nixosModules.hardware-hetzner-online-amd
     inputs.srvos.nixosModules.roles-nix-remote-builder
     self.nixosModules.holo-users
+    self.nixosModules.holo-users-interactive
 
     self.nixosModules.nix-build-distributor
 
@@ -20,6 +22,23 @@
     ../../nixos/shared.nix
     ../../nixos/shared-nix-settings.nix
     ../../nixos/shared-linux.nix
+
+    {
+      home-manager.users.dev = {pkgs, ...}: {
+        home.packages = [
+          # additional packages for this user go here
+        ];
+      };
+    }
+
+    ../../nixos/dev-minio.nix
+    {
+      services.devMinio.enable = true;
+    }
+  ];
+
+  nix.settings.system-features = [
+    "big-parallel"
   ];
 
   networking = {

modules/flake-parts/nixosModules.holo-users.nix

@@ -1,15 +1,100 @@
 {
+  self,
   inputs,
   lib,
   ...
-}: {
-  flake.nixosModules.holo-users = {
-    users.users.root.openssh.authorizedKeys = {
-      keyFiles =
-        lib.attrValues
-        (lib.filterAttrs (name: _: lib.hasPrefix "keys_" name) inputs);
-      keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICHujII5RAwfEXNBYxKhWv2Wx/oHeHUTc8CACZ3M5W3p neonphog@gmail.com"
+}: let
+  mkAuthorizedKeys = {...}: {
+    keyFiles =
+      lib.attrValues
+      (lib.filterAttrs (name: _: lib.hasPrefix "keys_" name) inputs);
+    keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICHujII5RAwfEXNBYxKhWv2Wx/oHeHUTc8CACZ3M5W3p neonphog@gmail.com"
+    ];
+  };
+in {
+  flake.nixosModules.holo-users = {config, ...}: {
+    users.mutableUsers = false;
+    users.users.root.openssh.authorizedKeys = mkAuthorizedKeys {};
+  };
+
+  flake.nixosModules.holo-users-interactive = {config, ...}: {
+    imports = [
+      inputs.home-manager.nixosModules.home-manager
+    ];
+
+    # a generic dev user that can be used to have per-host home-manager environments for it.
+    # this adds no risk since all potential users already have access to the root account via their SSH credentials.
+    users.users.dev = {
+      home = "/home/dev";
+      extraGroups = ["wheel"];
+      openssh.authorizedKeys = mkAuthorizedKeys {};
+      isNormalUser = true;
+      createHome = true;
+    };
+
+    sops.secrets.dev-age-key = {
+      sopsFile = self + "/secrets/dev/secrets.yaml";
+      owner = "dev";
+    };
+    home-manager = {
+      useGlobalPkgs = true;
+      useUserPackages = true;
+
+      sharedModules = [
+        inputs.sops-nix.homeManagerModules.sops
+      ];
+      users.dev = {pkgs, ...}: {
+        # Home Manager needs a bit of information about you and the
+        # paths it should manage.
+        home.username = "dev";
+        home.homeDirectory = "/home/dev";
+
+        home.packages = [
+          pkgs.coreutils
+          pkgs.neovim
+        ];
+
+        programs.bash.enable = true;
+        programs.bash.sessionVariables.SOPS_AGE_KEY_FILE = config.sops.secrets.dev-age-key.path;
+
+        programs.direnv.enable = true;
+        # TODO: enable this once home-manager is bumped to >= release-24.05
+        # programs.nix-direnv.enable = true;
+
+        # This value determines the Home Manager release that your
+        # configuration is compatible with. This helps avoid breakage
+        # when a new Home Manager release introduces backwards
+        # incompatible changes.
+        #
+        # You can update Home Manager without changing this value. See
+        # the Home Manager release notes for a list of state version
+        # changes in each release.
+        home.stateVersion = "23.11";
+
+        # Let Home Manager install and manage itself.
+        programs.home-manager.enable = true;
+
+        sops = {
+          age.keyFile = config.sops.secrets.dev-age-key.path;
+          defaultSopsFile = self + "/secrets/dev/secrets.yaml";
+        };
+      };
+    };
+
+    security.sudo = {
+      enable = true;
+      execWheelOnly = true;
+      extraRules = [
+        {
+          groups = ["wheel"];
+          commands = [
+            {
+              command = "ALL";
+              options = ["NOPASSWD"];
+            }
+          ];
+        }
       ];
     };
   };

modules/flake-parts/packages.zos-utils.nix

@@ -31,29 +31,16 @@
           '';
         };
 
-        # TODO: automate proper minio hosting. this is exemplary only and requires imperative setup of minio
-        zos-vm-serve-s3 = pkgs.writeShellApplication {
-          name = "zos-vm-serve-s3";
-          runtimeInputs = [
-            pkgs.minio
-          ];
-          text = ''
-            set -ueE -o pipefail
-
-            cd .minio
-
-            env \
-              MINIO_ROOT_USER=minioadmin \
-              MINIO_ROOT_PASSWORD="$(cat minioadmin.key)" \
-              minio server --console-address ":9001" storage
-          '';
-        };
-
         zos-vm-publish-s3 = let
-          s3BaseUrl = "sj-bm-hostkey0.dev.infra.holochain.org";
-          s3ListenUrl = "${s3BaseUrl}:9000";
-          s3HttpUrl = "https://${s3BaseUrl}/s3";
+          # TODO: document these: explain the relationship to the variables in the devShell's shellHook; if viable give them a common source of truth
+          s3BaseUrl = "dev.infra.holochain.org";
+          s3HttpUrl = "https://s3.${s3BaseUrl}";
+
+          # TODO: programmatically ensure this exists
           s3Bucket = "tfgrid-eval";
+
+          # TODO: this is faster however restricts publishing to the server itself
+          s3Alias = "devminio_local";
         in
           pkgs.writeShellApplication {
             name = "zos-vm-publish-s3";
@@ -73,12 +60,16 @@
               mkdir -p "$workDir"
               cd "$workDir"
 
+              s3_remote="''${MC_HOST_devminio_local:?}"
+              s3_remote="''${s3_remote/http:\/\//}"
+
               # mc rm --recursive --force localhost/${s3Bucket} || echo removal failed
               env RUST_MIN_STACK=8388608 \
-                rfs pack -m result.fl -s s3://minioadmin:"$(cat ../../.minio/minioadmin.key)"@${s3ListenUrl}/${s3Bucket}\?region=us-east-1 "$rootfs/" | tee rfs-pack.log
+                rfs pack -m result.fl -s "''${RFS_HOST_devminio:?}/${s3Bucket}?region=''${RFS_HOST_devminio_region:?}" "$rootfs/" | tee rfs-pack.log
+
+              mc cp result.fl ${s3Alias}/${s3Bucket}/"$rootfsBase".fl
 
-              # TODO: document or automate setting up the alias "localhost"
-              mc cp result.fl localhost/${s3Bucket}/"$rootfsBase".fl
+              # the final URL doesn't have the bucket name as it's implied as the default bucket.
               echo ${s3HttpUrl}/${s3Bucket}/"$rootfsBase".fl > public-url
 
               touch published
@@ -208,9 +199,12 @@
               kernel="$rootfs/boot/vmlinuz"
               initram="$rootfs/boot/initrd.img"
 
+              # FIXME: can't handle path ending in '/'
               workDir="$rootfs.work"
               mountDir="$workDir/mnt"
               mkdir -p "$mountDir"
+              # set it to read-only by default. the mount will be writable.
+              chmod 440 "$mountDir"
 
               socket="$workDir/virtiofs.sock"
 
@@ -219,7 +213,9 @@
                   exit 1
               }
 
-              rfs mount -m "$workDir"/result.fl "$mountDir" > "$workDir"/rfs_mount.log 2>&1 &
+              # FIXME: check whether the mount was successful
+              # FIXME: don't rely on sudo
+              sudo rfs mount -m "$workDir"/result.fl "$mountDir" 2>&1 | tee "$workDir"/rfs_mount.log &
               mountpid="$!"
 
               sleep 3
@@ -243,8 +239,8 @@
                   sudo kill "$fspid"
                   rm -rf "$socket"
 
-                  kill "$mountpid"
-                  umount --lazy "$mountDir"
+                  sudo kill "$mountpid"
+                  sudo umount --lazy "$mountDir"
                   rmdir "$mountDir"
                 )
               }