feat: introduce linux-builder-2 and migrate buildbot-nix-0

.sops.yaml

@@ -12,8 +12,9 @@ keys:
   - &dweb-reverse-proxy age1ygzy9clj0xavlmau0ham7j5nw8yy4z0q8hvkfpdgwc4fcr8nufpqrdxgvx
   - &linux-builder-01 age1kxkr407jz77ljrhgsfwfmv2yvqjprc6unvx389xp2f48xj8r0vqq2wew5r
   - &x64-linux-dev-01 age1vlxerq9j9jd00qvxj2gxds9re4dz2djqmllkhzsf44gz9a5y4ghs7807h9
+  - &linux-builder-2 age1w99tzxl88z7ct3ekpatl62wvhrx29pg450qmn822dpmz0evhxqxqy3scux
   - &tfgrid-shared age194xfar0gfdauu2dcxwqk9lh9d0vjfrzzs2ke0ppanpwv9eqxzs2qp7q7cn
-  - &buildbot-nix-0 age1w99tzxl88z7ct3ekpatl62wvhrx29pg450qmn822dpmz0evhxqxqy3scux
+  - &buildbot-nix-0 age1eel2m3jsanly3np5anytwyjze70v509mje2yu562e2k70ctdsvrqj23x20
   - &monitoring-0 age1cpcwv2wlszwase38zpngk6ld3vx8ev8jsv38m9pp74jvlutvxpsqt49yrr
   - &turn-0 age1yl2l760zjsxvrct97gtwfkfjlvhg3vkkwpud2usc35ktqnzhuurq3an2ns
   - &turn-1 age16thgapywna9zu3r87hfgvw097lq8r9z5rxunfjqmttcnpgxxcflqxkjycn
@@ -53,6 +54,7 @@ creation_rules:
           - *x64-linux-dev-01
           - *dweb-reverse-proxy
           - *linux-builder-01
+          - *linux-builder-2
           - *turn-0
           - *turn-1
           - *turn-2
@@ -64,6 +66,12 @@ creation_rules:
           - *linux-builder-01
           - *age_steveej
           - *age_r-vdp
+  - path_regex: ^secrets/linux-builder-2/[^/]+$
+    key_groups:
+      - age:
+          - *linux-builder-2
+          - *age_steveej
+          - *age_r-vdp
   - path_regex: ^secrets/x64-linux-dev-01/[^/]+$
     key_groups:
       - pgp:

modules/flake-parts/nixosConfigurations.buildbot-nix-0/configuration.nix

@@ -6,18 +6,8 @@
   lib,
   ...
 }:
-let
-  domainSuffix =
-    (builtins.elemAt
-      (builtins.attrValues self.nixosConfigurations.dweb-reverse-tls-proxy.config.services.bind.zones)
-      0
-    ).name;
-  appFqdn = "buildbot-nix-0.${domainSuffix}";
-  appId = 1008744;
-
-  oauthId = "Iv23liqmAiBw8ab9EF61";
-  topic = "holo-chain-buildbot-nix-0";
-in
+# Hetzner AX162-R #2497582
+# UEFI - factory
 {
   imports = [
     inputs.disko.nixosModules.disko
@@ -48,22 +38,31 @@ in
     domain = self.specialArgs.infraDomain;
     hostName = "buildbot-nix-0";
 
-    primaryIpv4 = "135.181.114.173";
-    primaryIpv6 = "2a01:4f9:4b:1a93::1/64";
+    primaryIpv4 = "65.109.100.254";
+    primaryIpv6 = "2a01:4f9:3080:25e7::1/64";
+
+    buildbot-nix = {
+      appFqdn = "buildbot-nix-0.${config.passthru.domain}";
+      appId = 1008744;
+      oauthId = "Iv23liqmAiBw8ab9EF61";
+      topic = "holo-chain-buildbot-nix-0";
+    };
   };
 
   networking = {
     inherit (config.passthru) hostName domain;
   };
   hostName = config.passthru.primaryIpv4;
 
-  nix.settings.max-jobs = 12;
+  nix.settings.max-jobs = 48;
 
   boot.loader.grub = {
-    efiSupport = false;
+    enable = true;
+    efiSupport = true;
+    efiInstallAsRemovable = true;
+    devices = [ "nodev" ];
   };
-  # boot.loader.systemd-boot.enable = true;
-  # boot.loader.efi.canTouchEfiVariables = true;
+  boot.loader.efi.canTouchEfiVariables = false;
   boot.kernelPackages = pkgs.linuxPackages_latest;
 
   systemd.network.networks."10-uplink".networkConfig.Address = config.passthru.primaryIpv6;
@@ -72,19 +71,38 @@ in
     let
       disk = id: {
         type = "disk";
-        device = "/dev/${id}";
+        device = "/dev/nvme${id}n1";
         content = {
           type = "gpt";
           partitions = {
-            boot = {
-              size = "1M";
-              type = "EF02"; # for grub MBR
+            ESP = {
+              priority = 100;
+              # Hetzner
+              start = "2M";
+              size = "500M";
+              # Hetzner's Debian installation was using "EFI System" as the partition code for the ESP mdadm raid1 members.
+              # so far _this_ is not working, however it did for Hetzner.
+              type = "EF00";
+              content = {
+                type = "mdraid";
+                name = "esp";
+              };
             };
-            mdadm = {
+
+            # boot = {
+            #   priority = 101;
+            #   size = "100%";
+            #   content = {
+            #     type = "mdraid";
+            #     name = "boot";
+            #   };
+            # };
+
+            rootfs = {
               size = "100%";
               content = {
                 type = "mdraid";
-                name = "raid0";
+                name = "rootfs";
               };
             };
           };
@@ -93,32 +111,53 @@ in
     in
     {
       disk = {
-        sda = disk "nvme0n1";
-        sdb = disk "nvme1n1";
+        sda = disk "0";
+        sdb = disk "1";
       };
+
       mdadm = {
-        raid0 = {
+        esp = {
+          type = "mdadm";
+          level = 1;
+          metadata = "1.0";
+          content = {
+            type = "filesystem";
+            # hetzner
+            format = "vfat";
+            extraArgs = [
+              "-F"
+              "16"
+            ];
+            # FIXME: it should be possible to use /boot/efi here and leave /boot on the btrfs
+            mountpoint = "/boot";
+            mountOptions = [ "umask=0077" ];
+          };
+        };
+
+        # boot = {
+        #   type = "mdadm";
+        #   level = 1;
+        #   content = {
+        #     type = "filesystem";
+        #     format = "ext3";
+        #     mountpoint = "/boot";
+        #   };
+        # };
+
+        rootfs = {
           type = "mdadm";
           level = 0;
           content = {
-            type = "gpt";
-            partitions = {
-              primary = {
-                size = "100%";
-                content = {
-                  type = "btrfs";
-                  extraArgs = [ "-f" ]; # Override existing partition
-                  subvolumes = {
-                    # Subvolume name is different from mountpoint
-                    "/rootfs" = {
-                      mountpoint = "/";
-                    };
-                    "/nix" = {
-                      mountOptions = [ "noatime" ];
-                      mountpoint = "/nix";
-                    };
-                  };
-                };
+            type = "btrfs";
+            extraArgs = [ "-f" ]; # Override existing partition
+            subvolumes = {
+              # Subvolume name is different from mountpoint
+              "/rootfs" = {
+                mountpoint = "/";
+              };
+              "/nix" = {
+                mountOptions = [ "noatime" ];
+                mountpoint = "/nix";
               };
             };
           };
@@ -142,7 +181,7 @@ in
     443
   ];
 
-  services.nginx.virtualHosts."${appFqdn}" = {
+  services.nginx.virtualHosts."${config.passthru.buildbot-nix.appFqdn}" = {
     enableACME = true;
     forceSSL = true;
   };
@@ -168,7 +207,7 @@ in
       # "x86_64-darwin"
       # "aarch64-darwin"
     ];
-    domain = appFqdn;
+    domain = config.passthru.buildbot-nix.appFqdn;
     outputsPath = "/var/www/buildbot/nix-outputs/";
     evalMaxMemorySize = 6 * 1024;
     evalWorkerCount = 8;
@@ -181,14 +220,14 @@ in
     # };
     github = {
       authType.app = {
-        id = appId;
+        id = config.passthru.buildbot-nix.appId;
         secretKeyFile = config.sops.secrets.buildbot-github-app-secret-key.path;
       };
       webhookSecretFile = config.sops.secrets.buildbot-github-webhook-secret.path;
       # this is a client secret
       oauthSecretFile = config.sops.secrets.buildbot-github-oauth-secret.path;
       # this is displayed in the app as "Client ID"
-      inherit oauthId topic;
+      inherit (config.passthru.buildbot-nix) oauthId topic;
     };
 
     postBuildSteps = [

modules/flake-parts/nixosConfigurations.dweb-reverse-tls-proxy/configuration.nix

@@ -214,6 +214,9 @@
       monitoring.${config.passthru.infraDomain}.               CNAME   monitoring-0.${config.passthru.infraDomain}.
 
       buildbot-nix-0.${config.passthru.infraDomain}.           A       ${self.nixosConfigurations.buildbot-nix-0.config.passthru.primaryIpv4}
+
+      linux-builder-01.${config.passthru.infraDomain}.         A       ${self.nixosConfigurations.linux-builder-01.config.passthru.primaryIpv4}
+      linux-builder-2.${config.passthru.infraDomain}.          A       ${self.nixosConfigurations.linux-builder-2.config.passthru.primaryIpv4}
     '';
   };