ao ci: two very basic functional tests

Signed-off-by: George Almasi <gheorghe@us.ibm.com>

.github/workflows/awstest.yml

@@ -27,8 +27,7 @@ jobs:
       # ---------------------------
       - name: install awscli and helm
         run: |
-          . ./util/awscli_util.sh
-          #awscli_install
+          . ./ci/util/util_awscli.sh
           awscli_config
           helm_install
         env:
@@ -41,37 +40,35 @@ jobs:
       # ---------------------------
       - name: create EC2 VM
         run: |
-          . ./util/awscli_util.sh
+          . ./ci/util/util_awscli.sh
           instanceid=$(awscli_launch ao)
           echo "instanceid=${instanceid}" >> ${GITHUB_ENV}
           awscli_wait_run ${instanceid} || exit -1
           echo "ipaddr=$(awscli_get_ipaddr ${instanceid})" >> ${GITHUB_ENV}
 
       # ---------------------------
-      # install minikube on the target VM
+      # install minikube on the target VM; set up access
       # ---------------------------
       - name: install and start minikube
         run: |
-          . ./util/awscli_util.sh
-          awscli_start_minikube ${ipaddr}
+          . ./ci/util/util_awscli.sh
+          awscli_start_minikube ${ipaddr} && \
+          awscli_access_minikube ${ipaddr}
 
       # ---------------------------
-      # download minikube credentials and open port access
+      # run all test scenarios
       # ---------------------------
-      - name: download minikube creds
+      - name: run test scenarios
         run: |
-          . ./util/awscli_util.sh
-          awscli_access_minikube ${ipaddr}
-
-
+          ./ci/tests/runtests.sh
 
       # ---------------------------
       # terminate the VM
       # ---------------------------
       - name: finish
         if: success() || failure()
         run: |
-          . ./util/awscli_util.sh
+          . ./ci/util/util_awscli.sh
           if [[ ${instanceid} != "" ]]
           then
               awscli_terminate ${instanceid}

ci/tests/1_simpletest/runtest.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+. ci/util/util_ao.sh
+
+ao_build      . || exit -1
+ao_clean      . || exit -1
+ao_deploy     . ${PWD}/ci/tests/1_simpletest/values.yml || exit -1
+ao_wait       . || exit -1
+ao_simpletest . || exit -1
+ao_clean      .

ci/tests/1_simpletest/values.yml

@@ -0,0 +1,13 @@
+tags:
+  init: true
+  registrar: true
+  verifier: true
+  agent: false
+  tenant: true
+
+global:
+  service:
+    registrar:
+      type: NodePort
+    verifier:
+      type: NodePort

ci/tests/2_privilegedagent/runtest.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+. ci/util/util_ao.sh
+
+ao_build      . || exit -1
+ao_clean      . || exit -1
+ao_deploy     . ${PWD}/ci/tests/2_privilegedagent/values.yml || exit -1
+ao_wait       . || exit -1
+ao_simpletest . || exit -1
+ao_clean      .

ci/tests/2_privilegedagent/values.yml

@@ -0,0 +1,19 @@
+tags:
+  init: true
+  registrar: true
+  verifier: true
+  agent: true
+  tenant: true
+
+global:
+  service:
+    agent:
+      privileged: true
+    registrar:
+      type: NodePort
+    verifier:
+      type: NodePort
+
+  configmap:
+    configParams:
+      KEYLIME_TENANT_REQUIRE_EK_CERT: false

ci/tests/runtests.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+passed=0
+failed=0
+total=0
+for test in `find tests -name runtest.sh -type f | sort`
+do
+    testdir=$(dirname ${test})
+    echo "RUNNING TEST: ${testdir}"
+    echo "---------------------"
+    if ${test}
+    then
+        passed=$((passed+1))
+    else
+        failed=$((failed+1))
+    fi
+    echo ""
+    total=$((total+1))
+done
+
+echo "+====================================+"
+printf "| Summary: %2d/%2d/%2d total/pass/fail  |\n" ${total} ${passed} ${failed}
+echo "+====================================+"
+
+exit ${failed}

ci/util/util_ao.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+
+# #########################
+# utility: clone AO main branch
+# used only when this script is invoked outside AO CI
+# #########################
+
+function ao_clone() {
+    local basedir=${1}
+    local aodir=${2}
+    local patchdir=${3}
+    local realpatchdir=$(realpath ${patchdir})
+    (cd ${basedir}
+     if ! test -d ${basedir}/${aodir}
+     then
+         echo -n "Checking out AO ..."
+         git clone https://github.com/keylime/attestation-operator ${aodir} > /tmp/ao-clone.log 2>&1
+         if [[ $? != 0 ]]
+         then
+             echo "ERROR: failed to checkout AO. Attaching log."
+             cat /tmp/ao-clone.log
+             exit -1
+         fi
+         echo "  done"
+         for f in $(find ${realpatchdir} -type f -name *.patch)
+         do
+             echo "Applying patches: ${f}"
+             (cd ${aodir}; cat ${f} | patch -f -p1) > /dev/null 2>&1
+         done
+     fi
+     return 0)
+}
+
+# #########################
+# utility: build the AO helm chart
+# #########################
+
+function ao_build() {
+    local aodir=${1}
+    (cd ${aodir}
+     echo -n "Building the helm chart ..."
+     make helm-build > /tmp/helm-build.log 2>&1
+     if [[ $? != 0 ]]
+     then
+         echo "\nERROR: helm build failed. Attaching log."
+         cat /tmp/helm-build.log
+         exit -1
+     fi
+     echo "done"
+     return 0)
+}
+
+
+# #########################
+# utility: deploy keylime with helm
+# NOTE makes a link 
+# #########################
+
+function ao_deploy() {
+    local aodir=${1}
+    local values=${2}
+    (cd ${aodir}
+     echo -n "Deploying keylime with helm ... "
+     make HELM_CHART_CUSTOM_VALUES=${values} \
+          helm-keylime-deploy > /tmp/helm-deploy.log 2>&1
+     if [[ $? != 0 ]]
+     then
+         echo "\nERROR: helm deploy failed. Attaching log."
+         cat /tmp/helm-deploy.log
+         exit -1
+     fi
+     echo "done"
+     return 0)
+}
+
+# #########################
+# step 6: wait until pods are running
+# #########################
+
+function ao_wait() {
+    local aodir=${1}
+    local podlist=${2:-"registrar tenant verifier"}
+    local timeout=${3:-300}
+    (cd ${aodir}
+     local t0=$(date +%s)
+     for comp in ${podlist}
+     do
+         echo -n "Waiting for ${comp} to be in run state: "
+         while ! kubectl get pods -n keylime --no-headers | grep ${comp} | grep Run > /dev/null 2>&1
+         do
+             local t1=$(date +%s)
+             if [[ ${t1} -gt $((t0+${timeout})) ]]
+             then
+                 echo "\nTIMED OUT."
+                 exit -1
+             fi
+             echo -n "."
+             sleep 5
+         done
+         echo "done"
+     done
+     echo "All components are running after $((t1-t0)) seconds."
+     return 0)     
+}
+
+# #########################
+# utility: clean up any previous deployments
+# #########################
+
+function ao_clean() {
+    local aodir=${1}
+    (cd ${aodir}
+     echo -n "Removing any previous deployments of keylime ... "
+     make helm-undeploy > /dev/null 2>&1
+     echo "done"
+     return 0)
+}
+
+
+# #########################
+# simple, stupid keylime test
+# #########################
+
+function ao_simpletest() {
+    local aodir=${1}
+    (cd ${aodir}
+     echo -n "Testing keylime function ... "
+     make helm-keylime-test > /tmp/keylime-test.log 2>&1
+     if [[ $? != 0 ]]
+     then
+         echo "\nERROR: test failed. Attaching log."
+         cat /tmp/keylime-test.log
+         exit -1
+     fi
+     echo "done"
+     return 0)     
+}
+
+

util/awscli_util.sh → ci/util/util_awscli.sh

@@ -51,19 +51,19 @@ function awscli_config() {
         echo "ERROR: AWS keypair secret undefined. Exiting."
         exit -1
     fi
-
+    
     if [[ "${AWS_ACCESS_KEY_ID}" == "" ]]
     then
         echo "ERROR: AWS access key ID undefined. Exiting."
         exit -1
     fi
-
+    
     if [[ "${AWS_ACCESS_KEY_SECRET}" == "" ]]
     then
         echo "ERROR: AWS secret undefined. Exiting."
         exit -1
     fi
-
+    
     # create ssh configuration and credentials
     mkdir ${HOME}/.ssh
     cat > ${HOME}/.ssh/config <<EOF
@@ -117,7 +117,7 @@ function awscli_launch() {
 	exit -1
     fi
     local instanceid=$(echo "${output}" | jq -r .Instances[0].InstanceId -)
-    aws ec2 create-tags --resources ${instanceid} --tags="Key=Name,Value=${vmname}-$$"  >/dev/null 2>&1
+    aws ec2 create-tags --resources ${instanceid} --tags="Key=Name,Value=${vmname}-$$"  >/dev/null 2>&1    
     echo ${instanceid}
     return 0
 }
@@ -241,7 +241,7 @@ EOF
     fi
     # install and start minikube
     echo "awscli_start_minikube on ${ipaddr}: installing minikube"
-    ssh -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr} > /tmp/minikube-install.log 2>&1 <<EOF
+    ssh -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr} > /tmp/minikube-install.log 2>&1 <<EOF    
 curl https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 -o /tmp/minikube-linux-amd64
 sudo mv /tmp/minikube-linux-amd64 /usr/local/bin/minikube
 sudo chmod 755 /usr/local/bin/minikube
@@ -297,8 +297,8 @@ function awscli_access_minikube() {
         echo "ERROR: failed to patch ${HOME}/.kube/config"
         exit -1
     fi
-
-
+    
+    
     # we don't need to worry about cleaning up this connection,
     # because the last step of any GH action is to remove the target VM itself.
     echo "awscli_access_minikube: creating a ssh tunnel to ${ipaddr}"
@@ -318,3 +318,5 @@ function awscli_access_minikube() {
     echo "awscli_access_minikube: SUCCESS after $((t1-t0)) seconds."
     return 0
 }
+
+