e2e.yaml

name: e2e

on:
  push:
    branches:
      - 'master'
    tags:
      - 'v*'
  pull_request:
    branches: [ master ]
  workflow_dispatch:

env:
  GO_TOOLCHAIN: "golang"
  # TODO: match BASEIMAGE with Makefile default (nonroot variant)
  BASEIMAGE: "gcr.io/distroless/static-debian11"
  KIND_CLUSTER_NAME: "kind"

jobs:
  build:
    name: build
    runs-on: ubuntu-20.04
    steps:
    - name: Check out code
      uses: actions/checkout@v4
    - name: Set up Go
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod
      id: go
    - name: Build
      run: |
        mkdir _output

        make docker-build/proxy-agent-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
        docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master
        docker save gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master > _output/konnectivity-agent.tar
        make docker-build/proxy-server-amd64 REGISTRY=gcr.io/k8s-staging-kas-network-proxy TAG=local BASEIMAGE=${{ env.BASEIMAGE }}
        docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:local gcr.io/k8s-staging-kas-network-proxy/proxy-server:master
        docker save gcr.io/k8s-staging-kas-network-proxy/proxy-server:master > _output/konnectivity-server.tar

    - uses: actions/upload-artifact@v4
      with:
        name: konnectivity-server
        path: _output/konnectivity-server.tar

    - uses: actions/upload-artifact@v4
      with:
        name: konnectivity-agent
        path: _output/konnectivity-agent.tar
  kind-e2e:
    name: kind-e2e
    runs-on: ubuntu-20.04
    timeout-minutes: 100
    needs:
    - build
    env:
      REGISTRY: gcr.io/k8s-staging-kas-network-proxy
      KIND_IMAGE: kindest/node:${{ matrix.k8s }}
      TAG: master
      CONNECTION_MODE: ${{ matrix.connection-mode }}
    strategy:
      fail-fast: false
      matrix:
        k8s: [ v1.28.7, v1.29.2, v1.30.4 ]
        connection-mode: [ grpc, http-connect ]
    steps:
    - name: Install kind
      run: |
        curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64
        chmod +x ./kind
        sudo mv ./kind /usr/local/bin/kind
    - name: Check out code
      uses: actions/checkout@v4
    - name: Set up Go
      uses: actions/setup-go@v5
      with:
        go-version-file: go.mod
      id: go
    - name: Download prebuilt konnectivity-server image
      uses: actions/download-artifact@v4
      with:
        name: konnectivity-server
    - name: Download prebuilt konnectivity-agent image
      uses: actions/download-artifact@v4
      with:
        name: konnectivity-agent
    - name: Load prebuilt konnectivity images
      run: |
        docker load --input konnectivity-server.tar
        docker load --input konnectivity-agent.tar
    - name: Fix konnectivity docker image tags
      run: |
        docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-server:master gcr.io/k8s-staging-kas-network-proxy/proxy-server-amd64:master
        docker tag gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master gcr.io/k8s-staging-kas-network-proxy/proxy-agent-amd64:master
    - name: Run e2e tests
      run: make test-e2e-ci
  e2e:
    name: e2e
    runs-on: ubuntu-20.04
    timeout-minutes: 100
    needs:
      - build
    strategy:
      fail-fast: false
      matrix:
        ipFamily: ["ipv4", "ipv6", "dual"]
        k8s: [ v1.28.7, v1.29.2, v1.30.4 ]
    env:
      JOB_NAME: "kindnetd-e2e-${{ matrix.ipFamily }}"
      IP_FAMILY: ${{ matrix.ipFamily }}
    steps:
    - name: Check out code
      uses: actions/checkout@v4

    - name: Enable ipv4 and ipv6 forwarding
      run: |
        sudo sysctl -w net.ipv6.conf.all.forwarding=1
        sudo sysctl -w net.ipv4.ip_forward=1

    - name: Set up environment (download dependencies)
      run: |
        TMP_DIR=$(mktemp -d)
        # Test binaries
        curl -L https://dl.k8s.io/${{ matrix.k8s }}/kubernetes-test-linux-amd64.tar.gz -o ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz
        tar xvzf ${TMP_DIR}/kubernetes-test-linux-amd64.tar.gz \
          --directory ${TMP_DIR} \
          --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test
        # kubectl
        curl -L https://dl.k8s.io/${{ matrix.k8s }}/bin/linux/amd64/kubectl -o ${TMP_DIR}/kubectl
        # kind
        curl -Lo ${TMP_DIR}/kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64
        # Install
        sudo cp ${TMP_DIR}/ginkgo /usr/local/bin/ginkgo
        sudo cp ${TMP_DIR}/e2e.test /usr/local/bin/e2e.test
        sudo cp ${TMP_DIR}/kubectl /usr/local/bin/kubectl
        sudo cp ${TMP_DIR}/kind /usr/local/bin/kind
        sudo chmod +x /usr/local/bin/*

    - name: Create multi node cluster
      run: |
        # output_dir
        mkdir -p _artifacts
        # create cluster
        cat <<EOF | /usr/local/bin/kind create cluster \
          --name ${{ env.KIND_CLUSTER_NAME}}           \
          --image kindest/node:${{ matrix.k8s }}  \
          -v7 --wait 1m --retain --config=-
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        networking:
          ipFamily: ${IP_FAMILY}
        nodes:
          - role: control-plane
            image: ${{ matrix.k8s }}
            kubeadmConfigPatchesJSON6902:
            - kind: ClusterConfiguration
              patch: |
                - op: add
                  path: /apiServer/certSANs/-
                  value: konnectivity-server.kube-system.svc.cluster.local
            kubeadmConfigPatches:
            - |
              kind: ClusterConfiguration
              apiServer:
                extraArgs:
                  "egress-selector-config-file": "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
                extraVolumes:
                - name: egress-selector-config-file
                  hostPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
                  mountPath: "/etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml"
                  readOnly: true
                - name: konnectivity-server
                  hostPath: "/etc/kubernetes/konnectivity-server"
                  mountPath: "/etc/kubernetes/konnectivity-server"
                  readOnly: true
            extraMounts:
            - hostPath: ./examples/kind/egress_selector_configuration.yaml
              containerPath: /etc/kubernetes/konnectivity-server-config/egress_selector_configuration.yaml
          - role: worker
            image: ${{ matrix.k8s }}
          - role: worker
            image: ${{ matrix.k8s }}
        EOF

        # dump the kubeconfig for later
        /usr/local/bin/kind get kubeconfig --name ${{ env.KIND_CLUSTER_NAME}} > _artifacts/kubeconfig.conf

    - uses: actions/download-artifact@v4
      with:
        name: konnectivity-server

    - uses: actions/download-artifact@v4
      with:
        name: konnectivity-agent

    - name: Install konnectivity
      run: |
        # preload konnectivity images
        docker load --input konnectivity-server.tar
        docker load --input konnectivity-agent.tar
        /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-server:master --name ${{ env.KIND_CLUSTER_NAME}}
        /usr/local/bin/kind load docker-image gcr.io/k8s-staging-kas-network-proxy/proxy-agent:master --name ${{ env.KIND_CLUSTER_NAME}}
        kubectl apply -f examples/kind/konnectivity-server.yaml
        kubectl apply -f examples/kind/konnectivity-agent-ds.yaml

    - name: Get Cluster status
      run: |
        # wait network is ready
        sleep 5
        /usr/local/bin/kubectl get nodes -o wide
        /usr/local/bin/kubectl get pods -A
        /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=kube-dns
        # smoke test
        /usr/local/bin/kubectl run test --image httpd:2
        /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods test
        /usr/local/bin/kubectl get pods -A -owide
        /usr/local/bin/kubectl wait --timeout=1m --for=condition=ready pods --namespace=kube-system -l k8s-app=konnectivity-agent
        /usr/local/bin/kubectl logs test
        

    - name: Workaround CoreDNS for IPv6 airgapped
      if: ${{ matrix.ipFamily == 'ipv6' }}
      run: |
        # Patch CoreDNS to work in Github CI
        # 1. Github CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
        # to work in an offline environment:
        # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
        # 2. Github CI adds following domains to resolv.conf search field:
        # .net.
        # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
        # otherwise pods stops trying to resolve the domain.
        # Get the current config
        original_coredns=$(/usr/local/bin/kubectl get -oyaml -n=kube-system configmap/coredns)
        echo "Original CoreDNS config:"
        echo "${original_coredns}"
        # Patch it
        fixed_coredns=$(
          printf '%s' "${original_coredns}" | sed \
            -e 's/^.*kubernetes cluster\.local/& net/' \
            -e '/^.*upstream$/d' \
            -e '/^.*fallthrough.*$/d' \
            -e '/^.*forward . \/etc\/resolv.conf$/d' \
            -e '/^.*loop$/d' \
        )
        echo "Patched CoreDNS config:"
        echo "${fixed_coredns}"
        printf '%s' "${fixed_coredns}" | /usr/local/bin/kubectl apply -f -

    - name: Run tests
      run: |
        export KUBERNETES_CONFORMANCE_TEST='y'
        export E2E_REPORT_DIR=${PWD}/_artifacts

        # Run tests
        /usr/local/bin/ginkgo --nodes=25                \
          --focus="\[Conformance\]"     \
          --skip="Feature|Federation|machinery|PerformanceDNS|DualStack|Disruptive|Serial|Slow|KubeProxy|LoadBalancer|GCE|Netpol|NetworkPolicy|NodeConformance"   \
          /usr/local/bin/e2e.test                       \
          --                                            \
          --kubeconfig=${PWD}/_artifacts/kubeconfig.conf     \
          --provider=local                              \
          --dump-logs-on-failure=false                  \
          --report-dir=${E2E_REPORT_DIR}                \
          --disable-log-dump=true

    - name: Upload Junit Reports
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: kind-junit-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }}
        path: './_artifacts/*.xml'

    - name: Export logs
      if: always()
      run: |
        /usr/local/bin/kind export logs --name ${KIND_CLUSTER_NAME} --loglevel=debug ./_artifacts/logs

    - name: Upload logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: kind-logs-${{ env.JOB_NAME }}-${{ matrix.k8s }}-${{ matrix.ipFamily }}-${{ github.run_id }}
        path: ./_artifacts/logs