diff --git a/.github/renovate.json b/.github/renovate.json index afeff6db9..08f5893dd 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -40,7 +40,7 @@ "kubernetes/.+\\.ya?ml$" ], "matchStrings": [ - "datasource=(?\\S+) depName=(?\\S+)( versioning=(?\\S+))?\n.*?\"(?.*)\"\n" + "datasource=(?\\S+) depName=(?\\S+)( registryUrl=(?\\S+))?\n.*?\"(?.*)\"\n" ], "datasourceTemplate": "{{#if datasource}}{{{datasource}}}{{else}}github-releases{{/if}}", "versioningTemplate": "{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}" diff --git a/.github/renovate/groups.json b/.github/renovate/groups.json index bcf97b9f2..daf8d9c4f 100644 --- a/.github/renovate/groups.json +++ b/.github/renovate/groups.json @@ -12,20 +12,11 @@ "separateMinorPatch": true }, { - "description": "Immich images", - "groupName": "Immich", - "matchPackagePatterns": ["immich"], - "matchDatasources": ["docker"], - "group": { - "commitMessageTopic": "{{{groupName}}} group" - }, - "separateMinorPatch": true - }, - { - "description": "Rook-Ceph image and chart", - "groupName": "Rook-Ceph", - "matchPackagePatterns": ["rook.ceph"], - "matchDatasources": ["docker", "helm"], + "description": "Flux Group", + "groupName": "Flux", + "matchPackagePatterns": ["flux"], + "matchDatasources": ["docker", "github-tags"], + "versioning": "semver", "group": { "commitMessageTopic": "{{{groupName}}} group" }, diff --git a/.taskfiles/ClusterTasks.yml b/.taskfiles/ClusterTasks.yml index 03d3c53cf..c5686ffbd 100644 --- a/.taskfiles/ClusterTasks.yml +++ b/.taskfiles/ClusterTasks.yml @@ -26,8 +26,7 @@ tasks: reconcile: desc: Force update Flux to pull in changes from your Git repository cmds: - - flux reconcile -n flux-system source git home-kubernetes - - flux reconcile -n flux-system kustomization cluster + - flux reconcile -n flux-system kustomization cluster --with-source hr-restart: desc: Restart all failed Helm Releases diff --git a/ansible/inventory/group_vars/kubernetes/k3s.yml b/ansible/inventory/group_vars/kubernetes/k3s.yml index 75b352d08..92cb40ab4 100644 --- a/ansible/inventory/group_vars/kubernetes/k3s.yml +++ b/ansible/inventory/group_vars/kubernetes/k3s.yml @@ -28,12 +28,9 @@ k3s_registration_address: "{{ kubevip_address }}" # (list) A list of URLs to deploy on the primary control plane. Read notes below. k3s_server_manifests_urls: - # Kube-vip + # Kube-vip RBAC - url: https://raw.githubusercontent.com/kube-vip/kube-vip/main/docs/manifests/rbac.yaml filename: custom-kube-vip-rbac.yaml - # Tigera Operator - - url: https://raw.githubusercontent.com/projectcalico/calico/v3.25.1/manifests/tigera-operator.yaml - filename: custom-calico-tigera-operator.yaml # Prometheus Operator - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml filename: custom-prometheus-alertmanagerconfigs.yaml @@ -56,8 +53,12 @@ k3s_server_manifests_urls: - url: https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.65.1/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml filename: custom-prometheus-prometheusagents.yaml -# (list) A flat list of templates to deploy on the primary control plane +# (list) A flat list of templates to deploy on the primary control plane nodes # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - - custom-calico-installation.yaml.j2 - - custom-kube-vip-daemonset.yaml.j2 + - cilium-helmchart.yaml.j2 + +# (list) A flat list of templates to deploy as static pods on all the control plane nodes +# /var/lib/rancher/k3s/agent/pod-manifests +k3s_server_pod_manifests_templates: + - kube-vip-static-pod.yaml.j2 diff --git a/ansible/inventory/group_vars/kubernetes/kube-vip.yml b/ansible/inventory/group_vars/kubernetes/kube-vip.yml deleted file mode 100644 index 1718ad9ce..000000000 --- a/ansible/inventory/group_vars/kubernetes/kube-vip.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -# (string) The ARP address kube-vip broadcasts -kubevip_address: "192.168.0.254" diff --git a/ansible/inventory/group_vars/kubernetes/os.yml b/ansible/inventory/group_vars/kubernetes/os.yml index 48f42f4f9..880dbcec3 100644 --- a/ansible/inventory/group_vars/kubernetes/os.yml +++ b/ansible/inventory/group_vars/kubernetes/os.yml @@ -1,25 +1,8 @@ --- -# (string) Timezone for the servers -# timezone: "America/New_York" # (list) Additional ssh public keys to add to the nodes # ssh_authorized_keys: -fedora: - packages: - - dnf-plugin-system-upgrade - - dnf-utils - - hdparm - - htop - - ipvsadm - - lm_sensors - - nano - - nvme-cli - - socat - - python3-kubernetes - - python3-libselinux - - python3-pyyaml - ubuntu: packages: - hdparm diff --git a/ansible/inventory/group_vars/kubernetes/supplemental.yml b/ansible/inventory/group_vars/kubernetes/supplemental.yml new file mode 100644 index 000000000..337099e05 --- /dev/null +++ b/ansible/inventory/group_vars/kubernetes/supplemental.yml @@ -0,0 +1,5 @@ +--- +timezone: "America/Los_Angeles" +kubevip_address: "10.69.69.2" +cluster_cidr: "10.98.0.0/16" +service_cidr: "10.99.0.0/16" diff --git a/ansible/inventory/group_vars/master/k3s.yml b/ansible/inventory/group_vars/master/k3s.yml index 14bda64f5..48ed222e3 100644 --- a/ansible/inventory/group_vars/master/k3s.yml +++ b/ansible/inventory/group_vars/master/k3s.yml @@ -26,6 +26,7 @@ k3s_server: - traefik disable-network-policy: true disable-cloud-controller: true + disable-kube-proxy: true write-kubeconfig-mode: "644" # Network CIDR to use for pod IPs cluster-cidr: "10.42.0.0/16" @@ -34,9 +35,6 @@ k3s_server: kube-controller-manager-arg: # Required to monitor kube-controller-manager with kube-prometheus-stack - "bind-address=0.0.0.0" - kube-proxy-arg: - # Required to monitor kube-proxy with kube-prometheus-stack - - "metrics-bind-address=0.0.0.0" kube-scheduler-arg: # Required to monitor kube-scheduler with kube-prometheus-stack - "bind-address=0.0.0.0" diff --git a/ansible/playbooks/cluster-installation.yml b/ansible/playbooks/cluster-installation.yml index e8bc8adeb..980dfebf6 100644 --- a/ansible/playbooks/cluster-installation.yml +++ b/ansible/playbooks/cluster-installation.yml @@ -45,53 +45,67 @@ regexp: "https://127.0.0.1:6443" replace: "https://{{ k3s_registration_address }}:6443" - - name: Resource Readiness Check + # Unmanaging and removing the Cilium HelmChart is required for + # flux to take over managing the lifecycle of Cilium + + - name: Post installation of custom manifests tasks run_once: true - kubernetes.core.k8s_info: - kubeconfig: /etc/rancher/k3s/k3s.yaml - kind: "{{ item.kind }}" - name: "{{ item.name }}" - namespace: "{{ item.namespace | default('') }}" - wait: true - wait_sleep: 10 - wait_timeout: 360 - loop: - - kind: Deployment - name: tigera-operator - namespace: tigera-operator - - kind: DaemonSet - name: kube-vip - namespace: kube-system - - kind: Installation - name: default - - kind: CustomResourceDefinition - name: alertmanagerconfigs.monitoring.coreos.com - - kind: CustomResourceDefinition - name: alertmanagers.monitoring.coreos.com - - kind: CustomResourceDefinition - name: podmonitors.monitoring.coreos.com - - kind: CustomResourceDefinition - name: probes.monitoring.coreos.com - - kind: CustomResourceDefinition - name: prometheuses.monitoring.coreos.com - - kind: CustomResourceDefinition - name: prometheusrules.monitoring.coreos.com - - kind: CustomResourceDefinition - name: servicemonitors.monitoring.coreos.com - - kind: CustomResourceDefinition - name: thanosrulers.monitoring.coreos.com - - kind: CustomResourceDefinition - name: scrapeconfigs.monitoring.coreos.com - - kind: CustomResourceDefinition - name: prometheusagents.monitoring.coreos.com when: - k3s_server_manifests_templates | length > 0 or k3s_server_manifests_urls | length > 0 - k3s_control_node is defined - k3s_control_node + block: + - name: Wait for custom manifests to rollout + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: "{{ item.kind }}" + name: "{{ item.name }}" + namespace: "{{ item.namespace | default('') }}" + wait: true + wait_sleep: 10 + wait_timeout: 360 + loop: + - name: cilium + kind: HelmChart + namespace: kube-system + - name: podmonitors.monitoring.coreos.com + kind: CustomResourceDefinition + - name: prometheusrules.monitoring.coreos.com + kind: CustomResourceDefinition + - name: servicemonitors.monitoring.coreos.com + kind: CustomResourceDefinition + - name: Wait for Cilium to rollout + kubernetes.core.k8s_info: + kubeconfig: /etc/rancher/k3s/k3s.yaml + kind: Job + name: helm-install-cilium + namespace: kube-system + wait: true + wait_condition: + type: Complete + status: true + wait_timeout: 360 + - name: Patch the Cilium HelmChart to unmanage it + kubernetes.core.k8s_json_patch: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + patch: + - op: add + path: /metadata/annotations/helmcharts.helm.cattle.io~1unmanaged + value: "true" + - name: Remove the Cilium HelmChart CR + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + name: cilium + kind: HelmChart + namespace: kube-system + state: absent # NOTE - # Cleaning up the manifests from the /var/lib/rancher/k3s/server/manifests directory + # Cleaning up certain manifests from the /var/lib/rancher/k3s/server/manifests directory # is needed because k3s has an awesome "feature" to always re-deploy them when the k3s # service is restarted. Removing them does not uninstall the manifests from your cluster. diff --git a/ansible/playbooks/cluster-nuke.yml b/ansible/playbooks/cluster-nuke.yml index e238e56a8..86766f4e4 100644 --- a/ansible/playbooks/cluster-nuke.yml +++ b/ansible/playbooks/cluster-nuke.yml @@ -9,7 +9,7 @@ - name: nuke prompt: |- Are you sure you want to nuke this cluster? - Type YES I WANT TO DESTROY THIS CLUSTER to proceed + Type 'YES I WANT TO DESTROY THIS CLUSTER' to proceed default: "n" private: false pre_tasks: @@ -22,6 +22,25 @@ ansible.builtin.pause: seconds: 5 tasks: + - name: Uninstall Cilium + when: + - k3s_control_node is defined + - k3s_control_node + ansible.builtin.shell: | + cilium uninstall --wait + environment: + KUBECONFIG: /etc/rancher/k3s/k3s.yaml + + - name: Prevent k3s from starting on reboot + ansible.builtin.systemd: + name: k3s + enabled: false + + - name: Reboot + ansible.builtin.reboot: + msg: Rebooting nodes + reboot_timeout: 3600 + - name: Uninstall k3s ansible.builtin.include_role: name: xanmanning.k3s diff --git a/ansible/playbooks/cluster-prepare.yml b/ansible/playbooks/cluster-prepare.yml index 8d89f5453..00cc334ab 100644 --- a/ansible/playbooks/cluster-prepare.yml +++ b/ansible/playbooks/cluster-prepare.yml @@ -21,25 +21,13 @@ - name: Networking | Set hostname to inventory hostname ansible.builtin.hostname: name: "{{ inventory_hostname }}" - - name: Networking | Update /etc/hosts to include inventory hostname + - name: Networking | Update hosts file to include inventory hostname ansible.builtin.blockinfile: path: /etc/hosts block: | 127.0.1.1 {{ inventory_hostname }} - - name: Packages | Fedora - block: - - name: Packages | Install required packages - ansible.builtin.dnf: - name: "{{ fedora.packages | default([]) }}" - state: present - update_cache: true - - name: Packages | Remove leaf packages - ansible.builtin.dnf: - autoremove: true - when: ansible_facts['distribution'] == 'Fedora' - - - name: Packages | Ubuntu + - name: Packages block: - name: Packages | Install required packages ansible.builtin.apt: @@ -55,7 +43,17 @@ install_recommends: false notify: Reboot when: "'raspi' in ansible_kernel" - when: ansible_facts['distribution'] == 'Ubuntu' + + - name: Packages | Cilium CLI + ansible.builtin.include_role: + name: githubixx.cilium_cli + public: true + vars: + cilium_cli_arch: "{{ 'amd64' if ansible_architecture == 'x86_64' else 'arm64' }}" + cilium_cli_tmp_directory: /tmp + when: + - k3s_control_node is defined + - k3s_control_node - name: User Configuration block: @@ -67,20 +65,12 @@ - name: System Configuration (1) block: - - name: System Configuration (1) | Disable firewalld | Fedora - ansible.builtin.systemd: - service: firewalld.service - enabled: false - masked: true - state: stopped - when: ansible_facts['distribution'] == 'Fedora' - - name: System Configuration (1) | Disable ufw | Ubuntu + - name: System Configuration (1) | Disable ufw ansible.builtin.systemd: service: ufw.service enabled: false masked: true state: stopped - when: ansible_facts['distribution'] == 'Ubuntu' - name: System Configuration (1) | Enable fstrim ansible.builtin.systemd: service: fstrim.timer @@ -117,28 +107,16 @@ net.bridge.bridge-nf-call-ip6tables: 1 fs.inotify.max_user_watches: 524288 fs.inotify.max_user_instances: 512 - - name: System Configuration (2) | Disable swap | Fedora - ansible.builtin.dnf: - name: zram-generator-defaults - state: absent - when: ansible_facts['distribution'] == 'Fedora' - - name: System Configuration (2) | Disable swap at runtime | Ubuntu + - name: System Configuration (2) | Disable swap at runtime ansible.builtin.command: swapoff -a when: - - ansible_facts['distribution'] == 'Ubuntu' - ansible_swaptotal_mb > 0 - - name: System Configuration (2) | Disable swap at boot | Ubuntu + - name: System Configuration (2) | Disable swap at boot ansible.posix.mount: name: "{{ item }}" fstype: swap state: absent loop: ["none", "swap"] - when: ansible_facts['distribution'] == 'Ubuntu' - - name: System Configuration (2) | Permissive SELinux | Fedora - ansible.posix.selinux: - state: permissive - policy: targeted - when: ansible_facts['distribution'] == 'Fedora' notify: Reboot handlers: diff --git a/ansible/playbooks/cluster-upgrade.yml b/ansible/playbooks/cluster-upgrade.yml new file mode 100644 index 000000000..a26df4028 --- /dev/null +++ b/ansible/playbooks/cluster-upgrade.yml @@ -0,0 +1,27 @@ +--- +- hosts: + - master + - worker + become: true + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Pausing for 5 seconds... + ansible.builtin.pause: + seconds: 5 + tasks: + - name: Ensure Kubernetes is running + ansible.builtin.include_role: + name: xanmanning.k3s + public: true + vars: + k3s_state: started + + - name: Upgrade kube-vip + when: + - k3s_control_node is defined + - k3s_control_node + ansible.builtin.template: + src: templates/kube-vip-static-pod.yaml.j2 + dest: "{{ k3s_server_pod_manifests_dir }}/kube-vip-static-pod.yaml" + mode: preserve diff --git a/ansible/playbooks/templates/cilium-helmchart.yaml.j2 b/ansible/playbooks/templates/cilium-helmchart.yaml.j2 new file mode 100644 index 000000000..bf889fce6 --- /dev/null +++ b/ansible/playbooks/templates/cilium-helmchart.yaml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cilium + namespace: kube-system +spec: + repo: https://helm.cilium.io/ + chart: cilium + # renovate: datasource=helm depName=cilium registryUrl=https://helm.cilium.io/ + version: 1.13.3 + targetNamespace: kube-system + bootstrap: true + valuesContent: |- + bpf: + masquerade: true + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + hubble: + enabled: false + ipam: + mode: kubernetes + k8sServiceHost: "{{ kubevip_address }}" + k8sServicePort: 6443 + kubeProxyReplacement: strict + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + operator: + enabled: true + replicas: 1 + rollOutPods: true + rollOutCiliumPods: true diff --git a/ansible/playbooks/templates/custom-calico-installation.yaml.j2 b/ansible/playbooks/templates/custom-calico-installation.yaml.j2 deleted file mode 100644 index ca127b7c0..000000000 --- a/ansible/playbooks/templates/custom-calico-installation.yaml.j2 +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: operator.tigera.io/v1 -kind: Installation -metadata: - name: default -spec: - registry: quay.io - imagePath: calico - calicoNetwork: - # https://docs.k3s.io/installation/network-options#custom-cni - containerIPForwarding: "Enabled" - # https://projectcalico.docs.tigera.io/networking/ip-autodetection - nodeAddressAutodetectionV4: - cidrs: - - "{{ (ansible_default_ipv4.network + '/' + ansible_default_ipv4.netmask) | ansible.utils.ipaddr('network/prefix') }}" - # Note: The ipPools section cannot be modified post-install. - ipPools: - - blockSize: 26 - cidr: "{{ k3s_server['cluster-cidr'] }}" - encapsulation: "VXLANCrossSubnet" - natOutgoing: Enabled - nodeSelector: all() - nodeMetricsPort: 9091 - typhaMetricsPort: 9093 diff --git a/ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 b/ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 deleted file mode 100644 index 94b0aa3ab..000000000 --- a/ansible/playbooks/templates/custom-kube-vip-daemonset.yaml.j2 +++ /dev/null @@ -1,72 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip -spec: - selector: - matchLabels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - template: - metadata: - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.5.12 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: address - value: "{{ k3s_registration_address }}" - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - serviceAccountName: kube-vip - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists diff --git a/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 b/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 new file mode 100644 index 000000000..60ea80657 --- /dev/null +++ b/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2 @@ -0,0 +1,57 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: kube-vip + namespace: kube-system + labels: + app.kubernetes.io/instance: kube-vip + app.kubernetes.io/name: kube-vip +spec: + containers: + - name: kube-vip + image: ghcr.io/kube-vip/kube-vip:v0.6.0 + imagePullPolicy: IfNotPresent + args: ["manager"] + env: + - name: address + value: "{{ kubevip_address }}" + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: prometheus_server + value: :2112 + securityContext: + capabilities: + add: ["NET_ADMIN", "NET_RAW"] + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - name: kubeconfig + hostPath: + path: /etc/rancher/k3s/k3s.yaml diff --git a/ansible/requirements.yml b/ansible/requirements.yml index f1c4c0ba4..e0d864988 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -11,6 +11,8 @@ collections: - name: kubernetes.core version: 2.4.0 roles: + - name: githubixx.cilium_cli + version: 2.6.0+0.13.2 - name: xanmanning.k3s src: https://github.com/PyratLabs/ansible-role-k3s.git - version: v3.4.1 + version: static-pods diff --git a/kubernetes/apps/kube-system/cilium/app/bgp-peering-policy.yaml b/kubernetes/apps/kube-system/cilium/app/bgp-peering-policy.yaml new file mode 100644 index 000000000..af2917f1f --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/bgp-peering-policy.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPPeeringPolicy +# comments courtesy of JJGadgets +# MAKE SURE CRDs ARE INSTALLED IN CLUSTER VIA cilium-config ConfigMap OR Cilium HelmRelease/values.yaml (bgpControlPlane.enabled: true), BEFORE THIS IS APPLIED! +# "CiliumBGPPeeringPolicy" Custom Resource will replace the old MetalLB BGP's "bgp-config" ConfigMap +# "CiliumBGPPeeringPolicy" is used with `bgpControlPlane.enabled: true` which uses GoBGP, NOT the old `bgp.enabled: true` which uses MetalLB +metadata: + name: bgp-loadbalancer-ip-main + namespace: kube-system +spec: + nodeSelector: + matchLabels: + kubernetes.io/os: "linux" # match all Linux nodes, change this to match more granularly if more than 1 PeeringPolicy is to be used throughout cluster + virtualRouters: + - localASN: 64512 + exportPodCIDR: false + serviceSelector: # this replaces address-pools, instead of defining the range of IPs that can be assigned to LoadBalancer services, now services have to match below selectors for their LB IPs to be announced + matchExpressions: + - { + key: thisFakeSelector, + operator: NotIn, + values: ["will-match-and-announce-all-services"], + } + neighbors: + - peerAddress: "10.0.0.1/32" # unlike bgp-config ConfigMap, peerAddress needs to be in CIDR notation + peerASN: 64512 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumLoadBalancerIPPool +metadata: + name: main-pool +spec: + cidrs: + - cidr: 10.100.100.0/24 diff --git a/kubernetes/apps/kube-system/cilium/app/helm-release.yaml b/kubernetes/apps/kube-system/cilium/app/helm-release.yaml new file mode 100644 index 000000000..f87034161 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/helm-release.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: cilium + namespace: kube-system +spec: + interval: 15m + chart: + spec: + chart: cilium + version: 1.13.3 + sourceRef: + kind: HelmRepository + name: cilium + namespace: flux-system + maxHistory: 2 + install: + createNamespace: true + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + bpf: + masquerade: true + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k3s/containerd/containerd.sock + hubble: + enabled: true + metrics: + enabled: + - dns:query;ignoreAAAA + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: true + relay: + enabled: true + rollOutPods: true + prometheus: + serviceMonitor: + enabled: true + ui: + enabled: true + rollOutPods: true + ingress: + enabled: true + className: nginx + hosts: + - &host "hubble.${SECRET_DOMAIN}" + tls: + - hosts: + - *host + ipam: + mode: kubernetes + k8sServiceHost: "${KUBE_VIP_ADDR}" + k8sServicePort: 6443 + kubeProxyReplacement: strict + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + operator: + replicas: 1 + rollOutPods: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + prometheus: + enabled: true + serviceMonitor: + enabled: true + rollOutCiliumPods: true diff --git a/kubernetes/apps/kube-system/cilium/app/kustomization.yaml b/kubernetes/apps/kube-system/cilium/app/kustomization.yaml new file mode 100644 index 000000000..9046e78f8 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: kube-system +resources: + - ./helm-release.yaml diff --git a/kubernetes/apps/kube-system/kube-vip/ks.yaml b/kubernetes/apps/kube-system/cilium/ks.yaml similarity index 67% rename from kubernetes/apps/kube-system/kube-vip/ks.yaml rename to kubernetes/apps/kube-system/cilium/ks.yaml index 5007cb7c3..289192c84 100644 --- a/kubernetes/apps/kube-system/kube-vip/ks.yaml +++ b/kubernetes/apps/kube-system/cilium/ks.yaml @@ -2,15 +2,15 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: cluster-apps-kube-vip + name: cluster-apps-cilium namespace: flux-system spec: - path: ./kubernetes/apps/kube-system/kube-vip/app + path: ./kubernetes/apps/kube-system/cilium/app prune: true sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false # no flux ks dependents interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/kube-system/kube-vip/app/daemonset.yaml b/kubernetes/apps/kube-system/kube-vip/app/daemonset.yaml deleted file mode 100644 index b22dab7d8..000000000 --- a/kubernetes/apps/kube-system/kube-vip/app/daemonset.yaml +++ /dev/null @@ -1,72 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-vip - namespace: kube-system - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip -spec: - selector: - matchLabels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - template: - metadata: - labels: - app.kubernetes.io/instance: kube-vip - app.kubernetes.io/name: kube-vip - spec: - containers: - - name: kube-vip - image: ghcr.io/kube-vip/kube-vip:v0.6.0 - imagePullPolicy: IfNotPresent - args: ["manager"] - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_cidr - value: "32" - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "false" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - - name: address - value: "${KUBE_VIP_ADDR}" - securityContext: - capabilities: - add: ["NET_ADMIN", "NET_RAW"] - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - serviceAccountName: kube-vip - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists diff --git a/kubernetes/apps/kube-system/kube-vip/app/kustomization.yaml b/kubernetes/apps/kube-system/kube-vip/app/kustomization.yaml deleted file mode 100644 index 1d0fbe56f..000000000 --- a/kubernetes/apps/kube-system/kube-vip/app/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: kube-system -resources: - - ./rbac.yaml - - ./daemonset.yaml -labels: - - pairs: - kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/kubernetes/apps/kube-system/kube-vip/app/rbac.yaml b/kubernetes/apps/kube-system/kube-vip/app/rbac.yaml deleted file mode 100644 index 3cd709682..000000000 --- a/kubernetes/apps/kube-system/kube-vip/app/rbac.yaml +++ /dev/null @@ -1,44 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-vip - namespace: kube-system -secrets: - - name: kube-vip ---- -apiVersion: v1 -kind: Secret -type: kubernetes.io/service-account-token -metadata: - name: kube-vip - namespace: kube-system - annotations: - kubernetes.io/service-account.name: kube-vip ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.kubernetes.io/autoupdate: "true" - name: system:kube-vip-role -rules: - - apiGroups: [""] - resources: ["services", "services/status", "nodes"] - verbs: ["list", "get", "watch", "update"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["list", "get", "watch", "update", "create"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: system:kube-vip-binding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-vip-role -subjects: - - kind: ServiceAccount - name: kube-vip - namespace: kube-system diff --git a/kubernetes/apps/kube-system/kustomization.yaml b/kubernetes/apps/kube-system/kustomization.yaml index aebb120b7..2d955540d 100644 --- a/kubernetes/apps/kube-system/kustomization.yaml +++ b/kubernetes/apps/kube-system/kustomization.yaml @@ -3,8 +3,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml + - ./cilium/ks.yaml - ./external-secrets/ks.yaml - - ./kube-vip/ks.yaml - ./local-path-provisioner/ks.yaml - ./metrics-server/ks.yaml - ./reloader/ks.yaml diff --git a/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml b/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml index 23daa240f..985be51e7 100644 --- a/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml +++ b/kubernetes/apps/kube-system/local-path-provisioner/ks.yaml @@ -12,7 +12,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/kube-system/metrics-server/ks.yaml b/kubernetes/apps/kube-system/metrics-server/ks.yaml index 5f8cedbf1..d10ca1fbe 100644 --- a/kubernetes/apps/kube-system/metrics-server/ks.yaml +++ b/kubernetes/apps/kube-system/metrics-server/ks.yaml @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/kube-system/reloader/ks.yaml b/kubernetes/apps/kube-system/reloader/ks.yaml index 68034b8a3..27a247c5b 100644 --- a/kubernetes/apps/kube-system/reloader/ks.yaml +++ b/kubernetes/apps/kube-system/reloader/ks.yaml @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/monitoring/grafana/ks.yaml b/kubernetes/apps/monitoring/grafana/ks.yaml index ef8412b7c..9e0507211 100644 --- a/kubernetes/apps/monitoring/grafana/ks.yaml +++ b/kubernetes/apps/monitoring/grafana/ks.yaml @@ -14,7 +14,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml index 928ea1955..a3bf26320 100644 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml @@ -15,7 +15,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/monitoring/kubernetes-dashboard/ks.yaml b/kubernetes/apps/monitoring/kubernetes-dashboard/ks.yaml index 34cd900c2..056079454 100644 --- a/kubernetes/apps/monitoring/kubernetes-dashboard/ks.yaml +++ b/kubernetes/apps/monitoring/kubernetes-dashboard/ks.yaml @@ -10,7 +10,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/networking/cloudflared/ks.yaml b/kubernetes/apps/networking/cloudflared/ks.yaml index ddc457b51..11e3a4711 100644 --- a/kubernetes/apps/networking/cloudflared/ks.yaml +++ b/kubernetes/apps/networking/cloudflared/ks.yaml @@ -12,7 +12,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/networking/ingress-nginx/ks.yaml b/kubernetes/apps/networking/ingress-nginx/ks.yaml index 228de0527..705d4e92d 100644 --- a/kubernetes/apps/networking/ingress-nginx/ks.yaml +++ b/kubernetes/apps/networking/ingress-nginx/ks.yaml @@ -31,7 +31,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/networking/k8s-gateway/ks.yaml b/kubernetes/apps/networking/k8s-gateway/ks.yaml index 607dc1660..c09d541ac 100644 --- a/kubernetes/apps/networking/k8s-gateway/ks.yaml +++ b/kubernetes/apps/networking/k8s-gateway/ks.yaml @@ -12,7 +12,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/system-upgrade/system-upgrade-controller/ks.yaml b/kubernetes/apps/system-upgrade/system-upgrade-controller/ks.yaml index a04405195..e4ff61911 100644 --- a/kubernetes/apps/system-upgrade/system-upgrade-controller/ks.yaml +++ b/kubernetes/apps/system-upgrade/system-upgrade-controller/ks.yaml @@ -28,7 +28,7 @@ spec: sourceRef: kind: GitRepository name: home-kubernetes - wait: true + wait: false interval: 30m retryInterval: 1m timeout: 5m diff --git a/kubernetes/apps/system-upgrade/system-upgrade-controller/plans/server.yaml b/kubernetes/apps/system-upgrade/system-upgrade-controller/plans/server.yaml index 25914319a..e3c21c132 100644 --- a/kubernetes/apps/system-upgrade/system-upgrade-controller/plans/server.yaml +++ b/kubernetes/apps/system-upgrade/system-upgrade-controller/plans/server.yaml @@ -12,6 +12,13 @@ spec: cordon: true nodeSelector: matchExpressions: - - {key: node-role.kubernetes.io/control-plane, operator: Exists} + - { key: node-role.kubernetes.io/control-plane, operator: Exists } + tolerations: + - { effect: NoSchedule, operator: Exists } + - { effect: NoExecute, operator: Exists } + - { key: node-role.kubernetes.io/control-plane, effect: NoSchedule, operator: Exists } + - { key: node-role.kubernetes.io/master, effect: NoSchedule, operator: Exists } + - { key: node-role.kubernetes.io/etcd, effect: NoExecute, operator: Exists } + - { key: CriticalAddonsOnly, operator: Exists } upgrade: image: rancher/k3s-upgrade diff --git a/kubernetes/flux/config/flux.yaml b/kubernetes/flux/config/flux.yaml index b86d8103c..3e64958da 100644 --- a/kubernetes/flux/config/flux.yaml +++ b/kubernetes/flux/config/flux.yaml @@ -8,7 +8,7 @@ spec: interval: 10m url: oci://ghcr.io/fluxcd/flux-manifests ref: - tag: v2.0.0-rc.3 + tag: v2.0.0-rc.5 --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/kubernetes/flux/repositories/helm/cilium.yaml b/kubernetes/flux/repositories/helm/cilium.yaml new file mode 100644 index 000000000..51c65d691 --- /dev/null +++ b/kubernetes/flux/repositories/helm/cilium.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: cilium + namespace: flux-system +spec: + interval: 1h + url: https://helm.cilium.io diff --git a/kubernetes/flux/repositories/helm/kustomization.yaml b/kubernetes/flux/repositories/helm/kustomization.yaml index 2e53154f6..d674e2071 100644 --- a/kubernetes/flux/repositories/helm/kustomization.yaml +++ b/kubernetes/flux/repositories/helm/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - ./bitnami.yaml - ./bjw-s.yaml + - ./cilium.yaml - ./external-dns.yaml - ./external-secrets.yaml - ./grafana.yaml diff --git a/kubernetes/flux/repositories/helm/prometheus-community.yaml b/kubernetes/flux/repositories/helm/prometheus-community.yaml index cc5d94076..a97a3d445 100644 --- a/kubernetes/flux/repositories/helm/prometheus-community.yaml +++ b/kubernetes/flux/repositories/helm/prometheus-community.yaml @@ -1,10 +1,10 @@ --- -# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/source.toolkit.fluxcd.io/helmrepository_v1beta2.json apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: name: prometheus-community namespace: flux-system spec: - interval: 2h - url: https://prometheus-community.github.io/helm-charts + type: oci + interval: 5m + url: oci://ghcr.io/prometheus-community/charts diff --git a/kubernetes/flux/vars/cluster-settings.yaml b/kubernetes/flux/vars/cluster-settings.yaml index 8b684e467..490eb16ad 100644 --- a/kubernetes/flux/vars/cluster-settings.yaml +++ b/kubernetes/flux/vars/cluster-settings.yaml @@ -5,11 +5,11 @@ metadata: name: cluster-settings namespace: flux-system data: + # Cluster Settings TIMEZONE: "America/Los_Angeles" - AIRGRADIENT_ADDR: "192.168.0.99" - KUBE_VIP_ADDR: "192.168.0.254" - METALLB_LB_RANGE: "192.168.0.240-192.168.0.253" - METALLB_K8S_GATEWAY_ADDR: "192.168.0.253" - METALLB_INGRESS_ADDR: "192.168.0.252" - METALLB_SMTP_ADDR: "192.168.0.250" - SYNOLOGY_NAS: "192.168.0.20" + CLUSTER_CIDR: "10.98.0.0/16" + SERVICE_CIDR: "10.99.0.0/16" + # Static Mappings + SYNOLOGY_NAS: "10.0.0.20" + AIRGRADIENT_ADDR: "10.42.42.10" + KUBE_VIP_ADDR: "10.69.69.2"