Add Minikube integration testing workflow (#160)

Move integration testing into pull_request action that is isolated to an action runner using minikube Co-authored-by: David Gamero <davidgamero@microsoft.com>
Azure · Dec 7, 2021 · 5cbd4ac · 5cbd4ac
1 parent 18ff120
commit 5cbd4ac
Show file tree

Hide file tree

Showing 16 changed files with 575 additions and 52 deletions.
diff --git a/.github/workflows/TriggerIntegrationTests.sh b/.github/workflows/TriggerIntegrationTests.sh
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
diff --git a/.github/workflows/run-integration-tests.yml b/.github/workflows/run-integration-tests.yml
@@ -0,0 +1,199 @@
+name: Minikube Integration Tests
+on: pull_request
+
+jobs:
+  run-integration-test:
+    name: Run Minikube Integration Tests
+    runs-on: ubuntu-latest
+    env:
+      KUBECONFIG: /home/runner/.kube/config
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Building latest changes
+        run: |
+          npm install
+          npm run build
+
+      - name: Set name of ns
+        run: echo "::set-output name=name::$(echo `date +%Y%m%d%H%M%S`)"
+        shell: bash
+        id: ns
+
+      - uses: Azure/setup-kubectl@v1
+        name: Install Kubectl
+
+      - id: setup-minikube
+        name: Setup Minikube
+        uses: manusa/actions-setup-minikube@v2.4.2
+        with:
+          minikube version: "v1.24.0"
+          kubernetes version: "v1.17.8"
+          driver: "none"
+        timeout-minutes: 3
+
+      - name: Create namespace to run tests
+        run: kubectl create ns test-${{ steps.ns.outputs.name }}
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: "3.x"
+
+      - name: Cleaning any previously created items
+        run: |
+          python test/integration/k8s-deploy-delete.py 'Service' 'nginx-service' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-delete.py 'Service' 'nginx-service-green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-delete.py 'Deployment' 'nginx-deployment-green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-delete.py 'Deployment' 'nginx-deployment' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-delete.py 'Ingress' 'nginx-ingress' ${{ steps.ns.outputs.name }}
+
+      - name: Executing deploy action
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.14.2
+          manifests: |
+            test/integration/manifests/test-service.yml
+          strategy: blue-green
+          route-method: service
+          action: deploy
+
+      - name: Checking if deploments and services were created with green labels
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment-green' 'green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service' 'green' ${{ steps.ns.outputs.name }}
+
+      - name: Executing promote action
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.14.2
+          manifests: |
+            test/integration/manifests/test-service.yml
+          strategy: blue-green
+          route-method: service
+          action: promote
+
+      - name: Checking if deploments and services were created with none labels after promote
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service' 'None' ${{ steps.ns.outputs.name }}
+
+      - name: Executing deploy action on
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.19.1
+          manifests: |
+            test/integration/manifests/test-service.yml
+          strategy: blue-green
+          route-method: service
+          action: deploy
+
+      - name: Checking if deploments and services were created with green labels, and old workloads persist on deploy
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment-green' 'green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service' 'green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment' 'None' ${{ steps.ns.outputs.name }}
+
+      - name: Executing reject action
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.19.1
+          manifests: |
+            test/integration/manifests/test-service.yml
+          strategy: blue-green
+          route-method: service
+          action: reject
+
+      - name: Checking if deploments and services were routed back to none labels after reject
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service' 'None' ${{ steps.ns.outputs.name }}
+
+      - name: Cleaning up current set up
+        run: |
+          python test/integration/k8s-deploy-delete.py 'Service' 'nginx-service' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-delete.py 'Deployment' 'nginx-deployment' ${{ steps.ns.outputs.name }}
+
+      - name: Executing deploy action for ingress
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.14.2
+          manifests: |
+            test/integration/manifests/test-ingress.yml
+          strategy: blue-green
+          route-method: ingress
+          action: deploy
+
+      - name: Checking if deploments, services and ingresses were created with green labels
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment-green' 'green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service-green' 'green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Ingress' 'nginx-ingress' 'green' ${{ steps.ns.outputs.name }}
+
+      - name: Executing promote action for ingress
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.14.2
+          manifests: |
+            test/integration/manifests/test-ingress.yml
+          strategy: blue-green
+          route-method: ingress
+          action: promote
+
+      - name: Checking if deploments, services and ingresses were created with none labels after promote
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Ingress' 'nginx-ingress' 'None' ${{ steps.ns.outputs.name }}
+
+      - name: Executing deploy action for ingress
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.19.1
+          manifests: |
+            test/integration/manifests/test-ingress.yml
+          strategy: blue-green
+          route-method: ingress
+          action: deploy
+
+      - name: Checking if deploments, services and ingresses were created with green labels after deploy, and old deployment persists
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment-green' 'green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service-green' 'green' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Ingress' 'nginx-ingress' 'green' ${{ steps.ns.outputs.name }}
+
+      - name: Executing reject action for ingress
+        uses: ./
+        with:
+          namespace: test-${{ steps.ns.outputs.name }}
+          images: nginx:1.19.1
+          manifests: |
+            test/integration/manifests/test-ingress.yml
+          strategy: blue-green
+          route-method: ingress
+          action: reject
+
+      - name: Checking if deploments, services and ingresses were created with none labels after reject
+        run: |
+          python test/integration/k8s-deploy-test.py 'Deployment' 'nginx-deployment' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Service' 'nginx-service' 'None' ${{ steps.ns.outputs.name }}
+          python test/integration/k8s-deploy-test.py 'Ingress' 'nginx-ingress' 'None' ${{ steps.ns.outputs.name }}
+
+      - if: ${{ always() }}
+        name: Delete created namespace
+        run: kubectl delete ns test-${{ steps.ns.outputs.name }}
+
+      - if: ${{ always() }}
+        name: Posting result back to PR
+        run: |
+          if [ '${{ steps.job-type.outputs.type }}' == 'pr' ]; then ruby postStatus.rb ${{github.event.client_payload.repository}} ${{github.event.client_payload.commit}} ${{secrets.L2_REPO_TOKEN}} ${{job.status}} ${{github.run_id}} ${{matrix.os}} false ${{ secrets.L2_REPO_USER }}; fi
+        shell: bash
diff --git a/.gitignore b/.gitignore
@@ -1 +1,3 @@
-node_modules
+node_modules
+
+.DS_Store
diff --git a/test/integration/k8s-deploy-delete.py b/test/integration/k8s-deploy-delete.py
@@ -0,0 +1,12 @@
+import subprocess, sys
+
+kind = sys.argv[1]
+name = sys.argv[2]
+namespace = 'test-' + sys.argv[3]
+
+try:
+  print('kubectl delete ' + kind + ' ' + name + ' -n ' + namespace)
+  deletion = subprocess.Popen(['kubectl', 'delete', kind, name, '--namespace', namespace])
+  result, err = deletion.communicate()
+except Exception as ex:
+  print('Error occured during deletion', ex)
diff --git a/test/integration/k8s-deploy-test.py b/test/integration/k8s-deploy-test.py
@@ -0,0 +1,33 @@
+import os, sys, json 
+
+RESULT = 'false'
+k8_object = None
+kind = sys.argv[1]
+name = sys.argv[2]
+color = sys.argv[3]
+namespace = 'test-' + sys.argv[4]
+
+print('kubectl get '+kind+' '+name+' -n '+namespace+' -o json')
+
+try:
+  k8_object = json.load(os.popen('kubectl get '+kind+' '+name+' -n '+namespace+' -o json'))
+except:
+  sys.exit(kind+' '+name+' not created')
+
+try:
+  if kind == 'Deployment' and k8_object['spec']['selector']['matchLabels']['k8s.deploy.color'] == str(color):
+    RESULT = 'true'
+  if kind == 'Service' and k8_object['spec']['selector']['k8s.deploy.color'] == str(color):
+    RESULT = 'true'
+  if kind == 'Ingress':
+    suffix = ''
+    if str(color) == 'green':
+      suffix = '-green'
+    if k8_object['spec']['rules'][0]['http']['paths'][0]['backend']['serviceName']=='nginx-service'+suffix and k8_object['spec']['rules'][0]['http']['paths'][1]['backend']['serviceName']=='unrouted-service':
+      RESULT = 'true'
+except:
+  pass    
+
+if RESULT=='false':
+  sys.exit(kind+' '+name+' not labelled properly')
+print('Test passed')
diff --git a/test/integration/manifests/test-ingress.yml b/test/integration/manifests/test-ingress.yml
@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.14.2
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1 
+kind: Service 
+metadata: 
+  name: nginx-service
+spec: 
+  selector: 
+    app: nginx
+  ports: 
+  - protocol: TCP 
+    port: 80 
+    targetPort: 80
+---
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: nginx-ingress
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /testpath
+        backend:
+          serviceName: nginx-service
+          servicePort: 80
+      - path: /testpath2
+        backend:
+          serviceName: unrouted-service
+          servicePort: 80
diff --git a/test/integration/manifests/test-service.yml b/test/integration/manifests/test-service.yml
@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-deployment
+  labels:
+    app: nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:1.14.2
+        ports:
+        - containerPort: 80
+---
+apiVersion: v1 
+kind: Service 
+metadata: 
+  name: nginx-service
+spec: 
+  selector: 
+    app: nginx
+  ports: 
+  - protocol: TCP 
+    port: 80 
+    targetPort: 80