Merge pull request #141 from eth-cscs/dev

Merge to master (version 1.8.3)

ci/k8s/Jenkinsfile

@@ -6,7 +6,7 @@ def vault_secrets = [
                         [envVar: 'REGISTRY_USER', vaultKey: 'REGISTRY_USER'],
                         [envVar: 'REPO_PREFIX', vaultKey: 'REPO_PREFIX'],
                         [envVar: 'K8S_CLUSTER_URL', vaultKey: 'K8S_CLUSTER_URL'],
-                        [envVar: 'firecrestci_github_access_token', vaultKey: 'firecrestci_github_access_token'], 
+                        [envVar: 'firecrestci_github_access_token', vaultKey: 'firecrestci_github_access_token'],
                     ]
                 ],
             ]
@@ -47,12 +47,12 @@ spec:
             steps {
                 container(name: 'kaniko') {
                     withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
-                    
+
                         sh '''
                             mkdir -p /kaniko/.docker
                             echo '{"auths":{"'$REGISTRY'":{"username":"'$REGISTRY_USER'","password":"'$JFROG_API_KEY'"}}}' > /kaniko/.docker/config.json
                         '''
-                }                
+                }
             }
           }
         }
@@ -68,7 +68,12 @@ spec:
                             ls -la
 
                             # build microservices
-                            for ms in certificator compute reservations status storage tasks utilities; do
+                            # Certificator separated since its base image is different (centos:7)
+                            /kaniko/executor --context ./ --dockerfile deploy/docker/certificator/Dockerfile \
+                                --destination $REPO_PREFIX/certificator:$GIT_COMMIT_SHORT --cleanup
+
+                            # Rest of the microservices
+                            for ms in compute reservations status storage tasks utilities; do
                                 /kaniko/executor --build-arg BASE_IMAGE=$REPO_PREFIX/f7t-base:latest --registry-mirror $REGISTRY \
                                 --context ./ --dockerfile deploy/docker/$ms/Dockerfile --destination $REPO_PREFIX/$ms:$GIT_COMMIT_SHORT --cleanup
                             done
@@ -96,14 +101,14 @@ spec:
 
                             export PATH=$PATH:$(pwd)/linux-amd64
                             helm list -n "$DEPLOY_NAMESPACE"
-                            
+
                             cd deploy/k8s
                             ls -la
                             echo "registry: $REPO_PREFIX\ntag: '$GIT_COMMIT_SHORT'\nnamespace: "$DEPLOY_NAMESPACE"\nregistry_secret_creds: registry-credentials\n" > values-dev.yaml
 
                             for app in config certificator client compute jaeger keycloak kong minio openapi reservations status storage tasks utilities; do
-                            helm uninstall "$app-env-dev" -n "$DEPLOY_NAMESPACE" || true
-                            helm install  --wait --wait-for-jobs --timeout 60s  "$app-env-dev" $app -n "$DEPLOY_NAMESPACE"  -f values-dev.yaml
+                              helm uninstall "$app-env-dev" -n "$DEPLOY_NAMESPACE" || true
+                              helm install  --wait --wait-for-jobs --timeout 60s  "$app-env-dev" $app -n "$DEPLOY_NAMESPACE"  -f values-dev.yaml
                             done
                             helm ls -n "$DEPLOY_NAMESPACE"
                         '''
@@ -116,13 +121,13 @@ spec:
             steps {
                 withVault([vaultSecrets: vault_secrets, configuration: vault_config]) {
                     withKubeConfig([credentialsId: 'firecrest-cicd-secret', serverUrl: K8S_CLUSTER_URL]) {
-                    
+
                         sh '''
                             curl -s -O https://get.helm.sh/helm-v3.7.1-linux-amd64.tar.gz
                             tar -xvf helm-v3.7.1-linux-amd64.tar.gz
                             export PATH=$PATH:$(pwd)/linux-amd64
                             helm list -n "$DEPLOY_NAMESPACE"
-                            
+
                             cd deploy/k8s
                             ls -la
 
@@ -158,22 +163,22 @@ spec:
                             helm list -n "$DEPLOY_NAMESPACE"
                             kubectl get pods -n "$DEPLOY_NAMESPACE"
 
-                            #cd ${BUILD_NUMBER}/firecrest
-
                             cd deploy/k8s
                             ls -la
                             echo "registry: $REPO_PREFIX\ntag: '$GIT_COMMIT_SHORT'\nnamespace: "$DEPLOY_NAMESPACE"\nregistry_secret_creds: registry-credentials\n" > values-dev.yaml
 
                             for use_gateway in True False; do
 
-                                helm uninstall tester-env-dev -n "$DEPLOY_NAMESPACE" || true
+                                helm uninstall tester-env-dev -n "$DEPLOY_NAMESPACE" && sleep 15s || true
 
                                 echo "Test using gateway: $use_gateway"
                                 helm install --wait --timeout 120s tester-env-dev tester -n "$DEPLOY_NAMESPACE"  -f values-dev.yaml --set tag=$GIT_COMMIT_SHORT \
                                     --set workingDir="/firecrest/src/tests/automated_tests" \
                                     --set use_gateway="$use_gateway" \
                                     --set pytest_config_file="firecrest-dev.ini"
 
+                                cont_exitcode=0
+
                                 while :
                                 do
                                     sleep 20s
@@ -182,12 +187,22 @@ spec:
                                     echo "Tester pod is: $tester_pod"
                                     pdstatus=$(kubectl get pods -n "$DEPLOY_NAMESPACE" $tester_pod -o jsonpath="{.status.phase}")
 
-                                    if [ "$pdstatus" = "Running" ]; then echo "$tester_pod is still $pdstatus"; continue; fi
+                                    if [ "$pdstatus" = "Running" ] || [ "$pdstatus" = "Pending" ]; then
+                                        cont_exitcode=$(kubectl get pods -n "$DEPLOY_NAMESPACE"  --selector=app=tester -o jsonpath="{.items[*].status.containerStatuses[1].state.terminated.exitCode}")
+
+                                        if [ "$cont_exitcode" = "" ]; then echo "$tester_pod is still $pdstatus"; continue; fi
+
+                                        cont_reason=$(kubectl get pods -n "$DEPLOY_NAMESPACE"  --selector=app=tester -o jsonpath="{.items[*].status.containerStatuses[1].state.terminated.reason}")
+                                        echo "Container tester exit code $cont_exitcode (reason: $cont_reason)";
+                                    fi
+
                                     kubectl logs $tester_pod -n firecrest-dev
-                                    if [ "$pdstatus" = "Failed" ]; then echo "$tester_pod has $pdstatus"; exit 1; fi
-                                    if [ "$pdstatus" = "Succeeded" ]; then echo "$tester_pod has $pdstatus"; break; fi
+                                    if [ "$cont_exitcode" = "0" ]; then
+                                        echo "$tester_pod success."; break;
+                                     else
+                                        echo "$tester_pod failed: $cont_exitcode"; exit 1;
+                                    fi
                                 done
-
                             done
                         '''
                     }
@@ -228,7 +243,7 @@ spec:
                             # create new openapi
                             cd deploy/k8s
                             ls -la
-                            
+
                             echo "registry: $REPO_PREFIX\ntag: tds\nnamespace: "$TDS_NAMESPACE"\nregistry_secret_creds: registry-credentials\n" > values-tds.yaml
 
                             helm install --wait --timeout 60s openapi-env-tds openapi -n "$TDS_NAMESPACE" -f values-tds.yaml
@@ -277,7 +292,7 @@ spec:
                             ./jfrog rt del --recursive --quiet --url="https://$REGISTRY/artifactory" --user="$REGISTRY_USER" --password="$JFROG_API_KEY" "$REGISTRY_GROUP/$ms/$GIT_COMMIT_SHORT/"
                         done
                     '''
-                }            
+                }
             }
         }
         success {

deploy/demo/docker-compose.yml

@@ -196,7 +196,7 @@ services:
 
   # complementary 3rd party services
   keycloak:
-    image: "jboss/keycloak:9.0.2"
+    image: "jboss/keycloak:15.0.2"
     container_name: fckeycloak
     env_file: keycloak/keycloak.env
     environment:

deploy/docker/base/Dockerfile

@@ -4,12 +4,7 @@
 ##  Please, refer to the LICENSE file in the root directory.
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
-from centos:7 as f7t-base
-
-# install epel repo for python-pip package
-RUN yum install -y epel-release
-RUN yum -y update
-RUN yum install -y python3-pip
+FROM python:3.8.12-slim
 
 RUN pip3 install --upgrade pip
 

deploy/docker/base/requirements.txt

@@ -1,5 +1,6 @@
 cryptography==3.4.6
-Flask==1.1.2
+markupsafe==2.0.1
+Flask==1.1.4
 PyJWT==1.7.1
 requests==2.22.0
 jaeger_client==4.5.0

deploy/docker/certificator/Dockerfile

@@ -4,14 +4,22 @@
 ##  Please, refer to the LICENSE file in the root directory.
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
-ARG BASE_IMAGE=f7t-base
+ARG BASE_IMAGE=centos:7
 from $BASE_IMAGE
 
-RUN yum install -y openssh-7.4p1
+RUN yum install -y epel-release
+RUN yum -y update
+RUN yum install -y python3-pip
+
+RUN pip3 install --upgrade pip
 
+ADD deploy/docker/base/requirements.txt base/requirements.txt
 ADD deploy/docker/certificator/requirements.txt deps/requirements.txt
+RUN pip3 install -r base/requirements.txt
 RUN pip3 install -r deps/requirements.txt
 
+RUN yum install -y openssh-7.4p1
+
 ADD src/certificator/certificator.py certificator.py
 
 ENTRYPOINT ["python3"]

deploy/docker/storage/requirements.txt

@@ -1,5 +1,5 @@
 -r ../base/requirements.txt
 keystoneauth1==4.3.0
-lxml==4.6.2
+lxml==4.6.5
 paramiko==2.6.0
 python-keystoneclient==4.2.0

deploy/docker/tester/Dockerfile

@@ -11,7 +11,7 @@
 ##        docker run -ti --rm -v $PWD:/firecrest f7t-tester
 ##        # (now inside the container run pytest as you want)
 ##        See scripts in ci folder.
-from python:3.8.5-slim
+from python:3.8.12-slim
 
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1

deploy/k8s/certificator/templates/deploy.certificator.yaml

@@ -36,8 +36,10 @@ items:
           envFrom:
             - configMapRef:
                 name: common-env-file
+          command: ["sh"]
+          args: ["-c", "cp /tmp/ca-key /ca-key; chmod 400 /ca-key; ls -l /; python3 certificator.py"]
           volumeMounts:
-          - mountPath: /ca-key #since ConfiMap mount
+          - mountPath: /tmp/ca-key #since ConfiMap mount
             name: f7t-cert-vol
             subPath: ca-key
           - mountPath: /user-key.pub

deploy/k8s/keycloak/templates/deploy.keycloak.yaml

@@ -17,7 +17,7 @@ items:
           app: keycloak
       spec:
         containers:
-        - image: jboss/keycloak:9.0.2
+        - image: jboss/keycloak:15.0.2
           env:
           - name: DB_VENDOR
             value: H2
@@ -38,6 +38,8 @@ items:
                 key: KEYCLOAK_PASSWORD
           - name: KEYCLOAK_USER
             value: admin
+          - name: JAVA_OPTS
+            value: " -Djboss.bind.address=0.0.0.0 -Djboss.bind.address.management=0.0.0.0 "
           name: f7t-keycloak
           ports:
           - containerPort: 8080

src/tests/automated_tests/integration/markers.py

@@ -7,4 +7,5 @@
 import os
 import pytest
 
-skipif_not_uses_gateway = pytest.mark.skipif(os.environ.get("USE_GATEWAY", "").lower() == "false", reason="This test uses the gateway to test microservice")
+skipif_uses_gateway = pytest.mark.skipif(os.environ.get("USE_GATEWAY", "").lower() == "true", reason="This test does not use the gateway to test microservice")
+skipif_not_uses_gateway = pytest.mark.skipif(os.environ.get("USE_GATEWAY", "").lower() == "false", reason="This test uses the gateway to test microservice")

src/tests/automated_tests/integration/test_storage.py

@@ -5,6 +5,7 @@
 #  SPDX-License-Identifier: BSD-3-Clause
 #
 import pytest
+import platform
 import requests
 import os
 import time
@@ -70,9 +71,10 @@ def test_post_upload_request(headers):
     msg = resp.json()["task"]["data"]["msg"]
     url = msg["parameters"]["url"] # "http://svc-minio:9000/service-account-firecrest-sample"
 
-    #ix = url.index("//")
-    #jx = url.index(":",ix)
-    #url=url.replace(url[ix+2:jx],"127.0.0.1")
+    if platform.system() == 'Darwin':
+        ix = url.index("//")
+        jx = url.index(":",ix)
+        url=url.replace(url[ix+2:jx],"127.0.0.1")
 
 
 

src/tests/template_client/Dockerfile

@@ -4,9 +4,9 @@
 ##  Please, refer to the LICENSE file in the root directory.
 ##  SPDX-License-Identifier: BSD-3-Clause
 ##
-FROM python:3.7-alpine
+FROM python:3.8.12-slim
 
-RUN pip install flask flask-WTF flask-bootstrap flask-oidc flask_sslify requests
+RUN pip install markupsafe==2.0.1 Flask==1.1.4 flask-WTF flask-bootstrap flask-oidc flask_sslify requests==2.22.0
 
 ADD ./ app