Merge pull request #96 from godatadriven/kg/configure-iptables-k3s

Change the way we configure DNS in k3s cluster

envs/api-python-s3-k8s/README.md

@@ -77,11 +77,11 @@ The KubernetesExecutor needs to know how to reach the cluster. This is done thro
 
 Furthermore the `AIRFLOW__KUBERNETES__POD_TEMPLATE_FILE` is used to configure the deployment of the worker pods.
 The same remote logging config and postgres connection config is used as the Airflow container uses. 
-Specifically for the running dag also a shared persistent volume is added to the pods to share temporary files with the downstream tasks.
+Specifically for the running dag a shared persistent volume is added to the pods to share temporary files with the downstream tasks.
 
 ## K3s Kubernetes cluster configuration
 
-To be able to connect to the docker images running S3, Mockserver and the postgres database a small patch to the kubedns configmap is needed. We add the hostnames and ip's as extra NodeHosts when airlfow starts. See `whirl.setup.d/03_patch_k3s_dns.sh`
+To be able to connect to the docker images running S3, Mockserver and the postgres database we need to change the NDS configuration of the k8s cluster a bit. we do this based on the example found in: https://github.com/corneliusludmann/k3s-docker-compose-dns where a custom entrypoint script adds IP tables rules to access Docker's internal DNS 127.0.0.11 from the k8s cluster.
 
 ## Kubectl from the Airflow docker container
 

envs/api-python-s3-k8s/docker-compose.yml

@@ -55,6 +55,7 @@ services:
 
   k3s-server:
     image: "rancher/k3s:${K3S_VERSION:-latest}"
+    entrypoint: /entrypoint
     command: 
       - server
       - "--disable"
@@ -76,6 +77,7 @@ services:
       - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig-k3s.yaml
       - K3S_KUBECONFIG_MODE=666
     volumes:
+      - ${ENVIRONMENT_FOLDER}/entrypoint.sh:/entrypoint
       - k3s-server:/var/lib/rancher/k3s
       # This is just so that we get the kubeconfig file out
       - ${ENVIRONMENT_FOLDER}/.kubeconfig:/output

envs/api-python-s3-k8s/entrypoint.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# entrypoint based on https://github.com/corneliusludmann/k3s-docker-compose-dns
+
+# Add IP tables rules to access Docker's internal DNS 127.0.0.11 from outside
+# based on https://serverfault.com/a/826424
+
+TCP_DNS_ADDR=$(iptables-save | grep DOCKER_OUTPUT | grep tcp | grep -o '127\.0\.0\.11:.*$')
+UDP_DNS_ADDR=$(iptables-save | grep DOCKER_OUTPUT | grep udp | grep -o '127\.0\.0\.11:.*$')
+
+iptables -t nat -A PREROUTING -p tcp --dport 53 -j DNAT --to "$TCP_DNS_ADDR"
+iptables -t nat -A PREROUTING -p udp --dport 53 -j DNAT --to "$UDP_DNS_ADDR"
+
+
+# Add this IP to resolv.conf since CoreDNS of k3s uses this file
+
+TMP_FILE=$(mktemp)
+sed "/nameserver.*/ a nameserver $(hostname -i | cut -f1 -d' ')" /etc/resolv.conf > "$TMP_FILE"
+cp "$TMP_FILE" /etc/resolv.conf
+rm "$TMP_FILE"
+
+# Custom entrypoint does not run k3s as PID 1: see https://github.com/k3s-io/k3s/issues/5123
+# In comment: https://github.com/k3s-io/k3s/issues/5123#issuecomment-1039674532 the script that fixes this
+# is mentioned: https://github.com/k3d-io/k3d/blob/main/pkg/types/fixes/assets/k3d-entrypoint-cgroupv2.sh
+# Copied that to this custom entrypoint script
+#########################################################################################################################################
+# DISCLAIMER                                                                                                                                                                                                                                                        #
+# Copied from https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/hack/dind#L28-L37                                                          #
+# Permission granted by Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> (https://github.com/k3d-io/k3d/issues/493#issuecomment-827405962)  #
+# Moby License Apache 2.0: https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/LICENSE                                                       #
+#########################################################################################################################################
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+  echo "[$(date -Iseconds)] [CgroupV2 Fix] Evacuating Root Cgroup ..."
+    # move the processes from the root group to the /init group,
+  # otherwise writing subtree_control fails with EBUSY.
+  mkdir -p /sys/fs/cgroup/init
+  busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+  # enable controllers
+  sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgroup/cgroup.subtree_control"
+  echo "[$(date -Iseconds)] [CgroupV2 Fix] Done"
+fi
+
+/bin/k3s "$@"

envs/api-python-s3-k8s/whirl.setup.d/03_k8s_persistant_volume.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+echo "==========================="
+echo " Prepare persistent volume "
+echo "==========================="
+cat <<EOFPV | /opt/airflow/kubectl --kubeconfig=/opt/airflow/.kubeconfig/k3s.yaml create -f -
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: local-data
+  labels:
+    type: local
+spec:
+  storageClassName: local-path
+  capacity:
+    storage: 1Gi
+  local:
+    path: /data/whirl
+  persistentVolumeReclaimPolicy: Retain
+  accessModes:
+    - ReadWriteMany
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - whirl-k3s-master
+EOFPV
+
+echo "================================"
+echo " Prepare persistent volumeclaim "
+echo "================================"
+cat <<EOFPVC | /opt/airflow/kubectl --kubeconfig=/opt/airflow/.kubeconfig/k3s.yaml create -f -
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: local-path-airflow-worker-pvc
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 1Gi
+EOFPVC

envs/dbt-example/whirl.setup.d/04_install_dbt.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 sudo apt-get install -y libsasl2-dev build-essential
-pip install dbt-core==1.4.5 airflow-dbt-python
+pip install dbt-core==1.7.5 airflow-dbt-python
 
 #airflow-dbt-python depends on the fs_default connection
 echo "====================================="