From 10955884146b2100ea433deaf9b0646df7b153e2 Mon Sep 17 00:00:00 2001 From: aserowy Date: Tue, 12 Nov 2024 13:39:20 +0100 Subject: [PATCH] tailspin: readd latest after build exceptions were resolved Signed-off-by: aserowy --- cluster/homelab/cilium.nix | 367 ++++++++++++++--------------- flake.lock | 6 +- home/modules/console/default.nix | 10 +- systems/homelab-01-nuc/default.nix | 10 +- systems/homelab-01-nuc/homelab.nix | 98 ++++++-- 5 files changed, 265 insertions(+), 226 deletions(-) diff --git a/cluster/homelab/cilium.nix b/cluster/homelab/cilium.nix index d1cccd39..4081ee47 100644 --- a/cluster/homelab/cilium.nix +++ b/cluster/homelab/cilium.nix @@ -15,214 +15,201 @@ values = { operator.replicas = 1; - # replicate k3s environment. - ipam.operator.clusterPoolIPv4PodCIDRList = [ "10.42.0.0/16" ]; + # # FIX: https://github.com/cilium/cilium/issues/31197 + # dnsProxy.enableTransparentMode = false; - k8sServiceHost = "localhost"; - k8sServicePort = 6444; + # # replicate k3s environment + # ipam = { + # mode = "kubernetes"; + # operator.clusterPoolIPv4PodCIDRList = [ "10.42.0.0/16" ]; + # }; - # policy enforcement. - policyEnforcementMode = "always"; - policyAuditMode = false; + # TODO: is host dependent: should come as modul option + k8sServiceHost = "192.168.178.53"; + k8sServicePort = 6443; - # set cilium as a kube-proxy replacement. + # policy enforcement + # policyEnforcementMode = "always"; + policyEnforcementMode = "never"; + + # set cilium as a kube-proxy replacement kubeProxyReplacement = true; - hubble = { - relay.enabled = true; - ui.enabled = true; - tls.auto.method = "cronJob"; - }; + # hubble = { + # relay.enabled = true; + # ui.enabled = true; + # }; }; }; resources = { ciliumNetworkPolicies = { - # Allow hubble relay server egress to nodes - allow-hubble-relay-server-egress.spec = { - description = "Policy for egress from hubble relay to hubble server in Cilium agent."; - endpointSelector.matchLabels."app.kubernetes.io/name" = "hubble-relay"; - egress = [ - { - toEntities = [ - "remote-node" - "host" - ]; - toPorts = [ - { - ports = [ - { - port = "4244"; - protocol = "TCP"; - } - ]; - } - ]; - } - ]; - }; - - # Allow hubble UI to talk to hubble relay - allow-hubble-ui-relay-ingress.spec = { - description = "Policy for ingress from hubble UI to hubble relay."; - endpointSelector.matchLabels."app.kubernetes.io/name" = "hubble-relay"; - ingress = [ - { - fromEndpoints = [ - { - matchLabels."app.kubernetes.io/name" = "hubble-ui"; - } - ]; - toPorts = [ - { - ports = [ - { - port = "4245"; - protocol = "TCP"; - } - ]; - } - ]; - } - ]; - }; - - # Allow hubble UI to talk to kube-apiserver - allow-hubble-ui-kube-apiserver-egress.spec = { - description = "Allow Hubble UI to talk to kube-apiserver"; - endpointSelector.matchLabels."app.kubernetes.io/name" = "hubble-ui"; - egress = [ - { - toEntities = [ "kube-apiserver" ]; - toPorts = [ - { - ports = [ - { - port = "6443"; - protocol = "TCP"; - } - ]; - } - ]; - } - ]; - }; - - # Allow kube-dns to talk to upstream DNS - allow-kube-dns-upstream-egress.spec = { - description = "Policy for egress to allow kube-dns to talk to upstream DNS."; - endpointSelector.matchLabels.k8s-app = "kube-dns"; - egress = [ - { - toEntities = [ "world" ]; - toPorts = [ - { - ports = [ - { - port = "53"; - protocol = "UDP"; - } - ]; - } - ]; - } - ]; - }; - - # Allow CoreDNS to talk to kube-apiserver - allow-kube-dns-apiserver-egress.spec = { - description = "Allow coredns to talk to kube-apiserver."; + allow-coredns-apiserver-egress.spec = { endpointSelector.matchLabels.k8s-app = "kube-dns"; egress = [ { toEntities = [ "kube-apiserver" ]; - toPorts = [ - { - ports = [ - { - port = "6443"; - protocol = "TCP"; - } - ]; - } - ]; - } - ]; - }; - - # Allow hubble-generate-certs job to talk to kube-apiserver - allow-hubble-generate-certs-apiserver-egress.spec = { - description = "Allow hubble-generate-certs job to talk to kube-apiserver."; - endpointSelector.matchLabels."batch.kubernetes.io/job-name" = "hubble-generate-certs"; - egress = [ - { - toEntities = [ "kube-apiserver" ]; - toPorts = [ - { - ports = [ - { - port = "6443"; - protocol = "TCP"; - } - ]; - } - ]; - } - ]; - }; - }; - - ciliumClusterwideNetworkPolicies = { - # Allow all cilium endpoints to talk egress to each other - allow-internal-egress.spec = { - description = "Policy to allow all Cilium managed endpoint to talk to all other cilium managed endpoints on egress"; - endpointSelector = { }; - egress = [ - { - toEndpoints = [ { } ]; - } - ]; - }; - - # Allow all health checks - cilium-health-checks.spec = { - endpointSelector.matchLabels."reserved:health" = ""; - ingress = [ - { - fromEntities = [ "remote-node" ]; - } - ]; - egress = [ - { - toEntities = [ "remote-node" ]; - } - ]; - }; - - # Allow all cilium managed endpoints to talk to cluster dns - allow-kube-dns-cluster-ingress.spec = { - description = "Policy for ingress allow to kube-dns from all Cilium managed endpoints in the cluster."; - endpointSelector.matchLabels = { - "k8s:io.kubernetes.pod.namespace" = "kube-system"; - "k8s-app" = "kube-dns"; - }; - ingress = [ - { - fromEndpoints = [ { } ]; - toPorts = [ - { - ports = [ - { - port = "53"; - protocol = "UDP"; - } - ]; - } - ]; } ]; }; }; + # ciliumNetworkPolicies = { + # allow-hubble-relay-server-egress.spec = { + # description = "Policy for egress from hubble relay to hubble server in Cilium agent."; + # endpointSelector.matchLabels."app.kubernetes.io/name" = "hubble-relay"; + # egress = [ + # { + # toEntities = [ + # "remote-node" + # "host" + # ]; + # toPorts = [ + # { + # ports = [ + # { + # port = "4244"; + # protocol = "TCP"; + # } + # ]; + # } + # ]; + # } + # ]; + # }; + # + # allow-hubble-ui-relay-ingress.spec = { + # description = "Policy for ingress from hubble UI to hubble relay."; + # endpointSelector.matchLabels."app.kubernetes.io/name" = "hubble-relay"; + # ingress = [ + # { + # fromEndpoints = [ + # { + # matchLabels."app.kubernetes.io/name" = "hubble-ui"; + # } + # ]; + # toPorts = [ + # { + # ports = [ + # { + # port = "4245"; + # protocol = "TCP"; + # } + # ]; + # } + # ]; + # } + # ]; + # }; + # + # allow-hubble-ui-kube-apiserver-egress.spec = { + # description = "Allow Hubble UI to talk to kube-apiserver"; + # endpointSelector.matchLabels."app.kubernetes.io/name" = "hubble-ui"; + # egress = [ + # { + # toEntities = [ "kube-apiserver" ]; + # toPorts = [ + # { + # ports = [ + # { + # port = "6443"; + # protocol = "TCP"; + # } + # ]; + # } + # ]; + # } + # ]; + # }; + # + # allow-kube-dns-upstream-egress.spec = { + # description = "Policy for egress to allow kube-dns to talk to upstream DNS."; + # endpointSelector.matchLabels.k8s-app = "kube-dns"; + # egress = [ + # { + # toEntities = [ "world" ]; + # toPorts = [ + # { + # ports = [ + # { + # port = "53"; + # protocol = "UDP"; + # } + # ]; + # } + # ]; + # } + # ]; + # }; + # + # allow-hubble-generate-certs-apiserver-egress.spec = { + # description = "Allow hubble-generate-certs job to talk to kube-apiserver."; + # endpointSelector.matchLabels."batch.kubernetes.io/job-name" = "hubble-generate-certs"; + # egress = [ + # { + # toEntities = [ "kube-apiserver" ]; + # toPorts = [ + # { + # ports = [ + # { + # port = "6443"; + # protocol = "TCP"; + # } + # ]; + # } + # ]; + # } + # ]; + # }; + # }; + # + # ciliumClusterwideNetworkPolicies = { + # allow-internal-egress.spec = { + # description = "Policy to allow all Cilium managed endpoint to talk to all other cilium managed endpoints on egress"; + # endpointSelector = { }; + # egress = [ + # { + # toEndpoints = [ { } ]; + # } + # ]; + # }; + # + # allow-kube-dns-cluster-ingress.spec = { + # description = "Policy for ingress allow to kube-dns from all Cilium managed endpoints in the cluster."; + # endpointSelector.matchLabels = { + # "k8s:io.kubernetes.pod.namespace" = "kube-system"; + # "k8s-app" = "kube-dns"; + # }; + # ingress = [ + # { + # fromEndpoints = [ { } ]; + # toPorts = [ + # { + # ports = [ + # { + # port = "53"; + # protocol = "UDP"; + # } + # ]; + # } + # ]; + # } + # ]; + # }; + # + # cilium-health-checks.spec = { + # endpointSelector.matchLabels."reserved:health" = ""; + # ingress = [ + # { + # fromEntities = [ "remote-node" ]; + # } + # ]; + # egress = [ + # { + # toEntities = [ "remote-node" ]; + # } + # ]; + # }; + # }; }; }; } diff --git a/flake.lock b/flake.lock index bf5ebe00..a42cc629 100644 --- a/flake.lock +++ b/flake.lock @@ -232,11 +232,11 @@ ] }, "locked": { - "lastModified": 1731045939, - "narHash": "sha256-YkDOBx0fBsuAe/RiKq0TCjvDFimQaEthLfK6jubWuXw=", + "lastModified": 1731414825, + "narHash": "sha256-vi90C+EzAsoTHQ05KZ94ZqnKZHZAWJR/Zsj+o+FT3vE=", "owner": "aserowy", "repo": "neocode", - "rev": "fe6caec2db70b7fe7c522220e08dd9524dad7b32", + "rev": "c859dd2cd40910fd72793348f40c011f19849096", "type": "github" }, "original": { diff --git a/home/modules/console/default.nix b/home/modules/console/default.nix index a5030bfb..f719449a 100644 --- a/home/modules/console/default.nix +++ b/home/modules/console/default.nix @@ -8,14 +8,6 @@ with lib; let cnfg = config.home.modules.console; - - system = pkgs.system; - pinned = import (builtins.fetchGit { - name = "tailspin_3_0_1"; - url = "https://github.com/NixOS/nixpkgs/"; - ref = "refs/heads/nixpkgs-unstable"; - rev = "05bbf675397d5366259409139039af8077d695ce"; - }) { inherit system; }; in { options.home.modules.console.enable = mkEnableOption "console"; @@ -40,7 +32,7 @@ in curl lazygit ncurses - pinned.tailspin + tailspin yeet fd diff --git a/systems/homelab-01-nuc/default.nix b/systems/homelab-01-nuc/default.nix index 48307942..997694d7 100644 --- a/systems/homelab-01-nuc/default.nix +++ b/systems/homelab-01-nuc/default.nix @@ -18,10 +18,18 @@ networking = { hostName = "homelab-01-nuc"; + interfaces = { + eno1.ipv4.addresses = [ + { + address = "192.168.178.53"; + prefixLength = 24; + } + ]; + }; + # enables wifi with: nmcli device wifi connect password networkmanager = { enable = true; - insertNameservers = [ "127.0.0.1" ]; }; }; diff --git a/systems/homelab-01-nuc/homelab.nix b/systems/homelab-01-nuc/homelab.nix index f4d4babb..6d9ef179 100644 --- a/systems/homelab-01-nuc/homelab.nix +++ b/systems/homelab-01-nuc/homelab.nix @@ -1,4 +1,9 @@ -{ config, ... }: +{ + config, + lib, + pkgs, + ... +}: { # FIX: for longhorn on nixos systemd.tmpfiles.rules = [ @@ -12,11 +17,17 @@ 22 80 443 - 6443 # k3s: required so that pods can reach the API server + 4240 + 6443 8080 ]; allowedUDPPorts = [ - 8472 # k3s, flannel: required if using multi-node for inter-node networking + 8472 + ]; + trustedInterfaces = [ + "cilium_host" + "cilium_net" + "cilium_vxlan" ]; }; }; @@ -25,29 +36,70 @@ k3s = { enable = true; role = "server"; - tokenFile = config.sops.secrets."k3s/cluster/token".path; - extraFlags = toString ([ - "--write-kubeconfig-mode \"0644\"" - "--cluster-init" - "--disable servicelb" - "--disable traefik" - "--disable local-storage" - # ] ++ (if meta.hostname == "homelab-0" then [ - # ] else [ - # "--server https://homelab-01-nuc:6443" - # ])); - ]); - # NOTE: meta comes from https://github.com/dreamsofautonomy/homelab/blob/main/nixos/flake.nix - # clusterInit = (meta.hostname == "homelab-0"); - clusterInit = true; + # tokenFile = config.sops.secrets."k3s/cluster/token".path; + extraFlags = + let + # admissionControlConfig = pkgs.writeText "k3s-admission-control-config.yaml" '' + # apiVersion: apiserver.config.k8s.io/v1 + # kind: AdmissionConfiguration + # plugins: + # - name: PodSecurity + # configuration: + # apiVersion: pod-security.admission.config.k8s.io/v1beta1 + # kind: PodSecurityConfiguration + # defaults: + # enforce: "baseline" + # enforce-version: "latest" + # audit: "restricted" + # audit-version: "latest" + # warn: "restricted" + # warn-version: "latest" + # exemptions: + # usernames: [] + # runtimeClasses: [] + # namespaces: [kube-system] + # ''; + # + serverConfig = pkgs.writeText "k3s-config.yaml" ( + lib.generators.toYAML { } { + #??? advertise-address = "192.168.178.53"; + + cluster-init = true; + # write-kubeconfig-mode = "0644"; + + # use persisted data directory + # data-dir = "/nix/persist/var/lib/rancher/k3s"; + + # instead cilium will be deployed + flannel-backend = "none"; + # disable-cloud-controller = true; + disable-kube-proxy = true; + disable-network-policy = true; + # disable-helm-controller = true; + + disable = [ + "traefik" + "servicelb" + # "local-storage" + "metrics-server" + ]; + + # kube-apiserver-arg = [ + # "admission-control-config-file=${admissionControlConfig}" + # "anonymous-auth=true" + # ]; + } + ); + in + "--config ${serverConfig}"; }; # NOTE: is used by longhorn - openiscsi = { - enable = true; - # name = "iqn.2016-04.com.open-iscsi:${meta.hostname}"; - name = "iqn.2016-04.com.open-iscsi:homelab-01-nuc"; - }; + # openiscsi = { + # enable = true; + # name = "iqn.2016-04.com.open-iscsi:${meta.hostname}"; + # name = "iqn.2016-04.com.open-iscsi:homelab-01-nuc"; + # }; }; sops = {