Merge branch 'master' into feat/team-project-cost-tracking

.github/workflows/image_build_push_squid.yaml

@@ -1,6 +1,7 @@
 name: Build Squid images
 
 on: 
+  workflow_dispatch:
   push:
     paths:
       - .github/workflows/image_build_push_squid.yaml

doc/README.md

@@ -85,7 +85,7 @@ For example - `gen3 help aws` opens `aws.md`
 * [utility vm](../tf_files/aws/modules/utility-vm/README.md)
 * [explorer infrastructure](https://github.com/uc-cdis/cdis-wiki/blob/master/dev/gen3/data_explorer/README.md)
 * [automation for gcp](../tf_files/gcp/commons/README.md)
-* [gcp bucket access flows for DCF](https://github.com/uc-cdis/fence/blob/master/docs/google_architecture.md)
+* [gcp bucket access flows for DCF](https://github.com/uc-cdis/fence/blob/master/docs/additional_documentation/google_architecture.md)
 * [authn and authz with fence](https://github.com/uc-cdis/fence/blob/master/README.md)
 * [jenkins](../kube/services/jenkins/README.md)
 * [jupyterhub configuration](../kube/services/jupyterhub/README.md)

files/squid_whitelist/web_whitelist

@@ -33,6 +33,7 @@ centos.mirrors.hoobly.com
 centos.mirrors.tds.net
 centos.mirrors.wvstateu.edu
 cernvm.cern.ch
+charts.authelia.com
 charts.helm.sh
 cloud.r-project.org
 coredns.github.io
@@ -124,6 +125,7 @@ neuro.debian.net
 neurodeb.pirsquared.org
 nginx.org
 nvidia.github.io
+ohsu-comp-bio.github.io
 opportunityinsights.org
 orcid.org
 pgp.mit.edu

files/squid_whitelist/web_wildcard_whitelist

@@ -59,6 +59,7 @@
 .immport.org
 .jenkins.io
 .jenkins-ci.org
+.jetstack.io
 .k8s.io
 .kegg.jp
 .kidsfirstdrc.org

gen3/bin/kube-setup-aws-es-proxy.sh

@@ -21,7 +21,7 @@ if g3kubectl get secrets/aws-es-proxy > /dev/null 2>&1; then
     if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${esDomain}"  --query "DomainStatusList[*].Endpoints" --output text)" \
         && [[ -n "${ES_ENDPOINT}" && -n "${esDomain}" ]]; then
       gen3 roll aws-es-proxy GEN3_ES_ENDPOINT "${ES_ENDPOINT}"
-           g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
+      g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
       g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-service.yaml"
       gen3_log_info "kube-setup-aws-es-proxy" "The aws-es-proxy service has been deployed onto the k8s cluster."
     else
@@ -33,8 +33,8 @@ if g3kubectl get secrets/aws-es-proxy > /dev/null 2>&1; then
       gen3 kube-setup-networkpolicy service aws-es-proxy
       g3kubectl patch deployment "aws-es-proxy-deployment" -p  '{"spec":{"template":{"metadata":{"labels":{"netvpc":"yes"}}}}}' || true
     fi
-  elif [ "$es7" = true ]; then
-    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata-2 --query "DomainStatusList[*].Endpoints" --output text)" \
+  elif [ "$es7" = false ]; then
+    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata --query "DomainStatusList[*].Endpoints" --output text)" \
         && [[ -n "${ES_ENDPOINT}" && -n "${envname}" ]]; then
       gen3 roll aws-es-proxy GEN3_ES_ENDPOINT "${ES_ENDPOINT}"
       g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
@@ -50,9 +50,10 @@ if g3kubectl get secrets/aws-es-proxy > /dev/null 2>&1; then
       g3kubectl patch deployment "aws-es-proxy-deployment" -p  '{"spec":{"template":{"metadata":{"labels":{"netvpc":"yes"}}}}}' || true
     fi
   else
-    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata --query "DomainStatusList[*].Endpoints" --output text)" \
+    if ES_ENDPOINT="$(aws es describe-elasticsearch-domains --domain-names "${envname}"-gen3-metadata-2 --query "DomainStatusList[*].Endpoints" --output text)" \
         && [[ -n "${ES_ENDPOINT}" && -n "${envname}" ]]; then
       gen3 roll aws-es-proxy GEN3_ES_ENDPOINT "${ES_ENDPOINT}"
+      g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-priority-class.yaml"
       g3kubectl apply -f "${GEN3_HOME}/kube/services/aws-es-proxy/aws-es-proxy-service.yaml"
       gen3_log_info "kube-setup-aws-es-proxy" "The aws-es-proxy service has been deployed onto the k8s cluster."
     else

gen3/bin/kube-setup-ingress.sh

@@ -15,23 +15,6 @@ ctxNamespace="$(g3kubectl config view -ojson | jq -r ".contexts | map(select(.na
 scriptDir="${GEN3_HOME}/kube/services/ingress"
 
 gen3_ingress_setup_waf() {
-    gen3_log_info "Starting GPE-312 waf setup"
-    #variable to see if WAF already exists
-    export waf=`aws wafv2 list-web-acls --scope REGIONAL | jq -r '.WebACLs[]|select(.Name| contains(env.vpc_name)).Name'`
-if [[ -z $waf ]]; then
-    gen3_log_info "Creating Web ACL. This may take a few minutes."
-    aws wafv2 create-web-acl\
-        --name $vpc_name-waf \
-        --scope REGIONAL \
-        --default-action Allow={} \
-        --visibility-config SampledRequestsEnabled=true,CloudWatchMetricsEnabled=true,MetricName=GPE-312WebAclMetrics \
-        --rules file://${GEN3_HOME}/gen3/bin/waf-rules-GPE-312.json \
-        --region us-east-1
-    #Need to sleep to avoid "WAFUnavailableEntityException" error since the waf takes a bit to spin up
-    sleep 300
-else
-    gen3_log_info "WAF already exists. Skipping..."
-fi
     gen3_log_info "Attaching ACL to ALB."
     export acl_arn=`aws wafv2 list-web-acls --scope REGIONAL | jq -r '.WebACLs[]|select(.Name| contains(env.vpc_name)).ARN'`
     export alb_name=`kubectl get ingress gen3-ingress | awk '{print $4}' | tail +2 |  sed 's/^\([A-Za-z0-9]*-[A-Za-z0-9]*-[A-Za-z0-9]*\).*/\1/;q'`

kube/services/audit-service/audit-service-deploy.yaml

@@ -94,6 +94,11 @@ spec:
             readOnly: true
             mountPath: "/src/audit-service-config.yaml"
             subPath: "audit-service-config.yaml"
+          # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+          - name: "config-volume"
+            readOnly: true
+            mountPath: "/audit/audit-service-config.yaml"
+            subPath: "audit-service-config.yaml"
         resources:
           requests:
             cpu: 100m
@@ -109,6 +114,11 @@ spec:
               readOnly: true
               mountPath: "/src/audit-service-config.yaml"
               subPath: "audit-service-config.yaml"
+            # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+            - name: "config-volume"
+              readOnly: true
+              mountPath: "/audit/audit-service-config.yaml"
+              subPath: "audit-service-config.yaml"
           resources:
             limits:
               cpu: 0.8
@@ -117,4 +127,5 @@ spec:
           args:
             - "-c"
             - |
-              /env/bin/alembic upgrade head
+              # Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility 
+              poetry run alembic upgrade head || /env/bin/alembic upgrade head

kube/services/jobs/indexd-userdb-job.yaml

@@ -74,7 +74,7 @@ spec:
             # Script always succeeds if it runs (echo exits with 0)
             # indexd image does not include jq, so use python 
             - |
-              eval $(python 2> /dev/null <<EOM
+              eval $( (python 2> /dev/null || poetry run python 2> /dev/null) <<EOM
               import json
 
               sheepdog_data = json.load(open('/var/www/indexd/sheepdog_creds.json', 'r'))
@@ -90,7 +90,7 @@ spec:
               EOM
               )
               for user in "${!user_db[@]}"; do
-                python /indexd/bin/index_admin.py create --username "$user" --password "${user_db[$user]}"
+                (python /indexd/bin/index_admin.py create --username "$user" --password "${user_db[$user]}" 2> /dev/null || poetry run python /indexd/bin/index_admin.py create --username "$user" --password "${user_db[$user]}")
               done
               echo "Exit code: $?"
       restartPolicy: Never

kube/services/jobs/psql-db-copy-aurora-job.yaml

@@ -168,11 +168,11 @@ spec:
                 gen3_log_info "Source DB: $source_db_database, Username: $db_username, Current DB: $db_database, Target DB: $target_db"
 
                 # DB commands
-                gen3 psql aurora -c "GRANT $db_username TO $aurora_master_username"
+                gen3 psql aurora -c "GRANT \"$db_username\" TO \"$aurora_master_username\""
                 gen3 psql aurora -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$source_db_database' AND pid <> pg_backend_pid()"
-                gen3 psql aurora -c "CREATE DATABASE $target_db WITH TEMPLATE $source_db_database OWNER $db_username"
-                pg_command="DO \$\$ DECLARE tbl record; BEGIN FOR tbl IN (SELECT table_schema || '.' || table_name AS full_table_name FROM information_schema.tables WHERE table_schema = 'public') LOOP EXECUTE 'ALTER TABLE ' || tbl.full_table_name || ' OWNER TO $db_username;'; END LOOP; END \$\$;"
-                PGPASSWORD=${aurora_master_password} psql -h $aurora_host_name -U $aurora_master_username -d "$target_db" -c "$pg_command"
+                gen3 psql aurora -c "CREATE DATABASE \"$target_db\" WITH TEMPLATE \"$source_db_database\" OWNER \"$db_username\""
+                pg_command="DO \$\$ DECLARE tbl record; BEGIN FOR tbl IN (SELECT table_schema || '.' || table_name AS full_table_name FROM information_schema.tables WHERE table_schema = 'public') LOOP EXECUTE 'ALTER TABLE ' || tbl.full_table_name || ' OWNER TO \"$db_username\";'; END LOOP; END \$\$;"
+                PGPASSWORD=${aurora_master_password} psql -h $aurora_host_name -U "$aurora_master_username" -d "$target_db" -c "$pg_command"
                 if [ $? -eq 0 ]; then
                   gen3_log_info "Successfully processed $database"
                   new_databases+=("$target_db")

kube/services/metadata/metadata-deploy.yaml

@@ -117,6 +117,11 @@ spec:
             readOnly: true
             mountPath: /src/.env
             subPath: metadata.env
+          # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+          - name: config-volume-g3auto
+            readOnly: true
+            mountPath: /mds/.env
+            subPath: metadata.env
           - name: config-volume
             readOnly: true
             mountPath: /aggregate_config.json
@@ -140,6 +145,11 @@ spec:
               readOnly: true
               mountPath: /src/.env
               subPath: metadata.env
+            # Added an additional volume mount for new images using the /<app-name> directory, while retaining the 'src' mount for backward compatibility.
+            - name: config-volume-g3auto
+              readOnly: true
+              mountPath: /mds/.env
+              subPath: metadata.env
           resources:
             limits:
               cpu: 0.8
@@ -148,4 +158,6 @@ spec:
           args:
             - "-c"
             - |
-              /env/bin/alembic upgrade head
+              # Managing virtual environments via poetry instead of python since the AL base image update, but retaining backwards compatibility 
+              poetry run alembic upgrade head || /env/bin/alembic upgrade head
+

kube/services/node-affinity-daemonset/README.md

@@ -0,0 +1,5 @@
+# Prerequisites
+
+This service needs certmanager to work. Please install certmanager before deploying this service. Once certmanager is installed, you can deploy this service by applying the manifests in this directory.
+
+Code lives in https://github.com/uc-cdis/node-affinity-webhook/

kube/services/node-affinity-daemonset/deployment.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: node-affinity-daemonset
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: node-affinity-daemonset
+  template:
+    metadata:
+      labels:
+        app: node-affinity-daemonset
+    spec:
+      containers:
+        - name: node-affinity-daemonset
+          image: quay.io/cdis/node-affinity-daemonset:master
+          ports:
+            - containerPort: 8443
+          volumeMounts:
+            - name: webhook-certs
+              mountPath: /etc/webhook/certs
+              readOnly: true
+      volumes:
+        - name: webhook-certs
+          secret:
+            secretName: webhook-certs #pragma: allowlist secret

kube/services/node-affinity-daemonset/service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: node-affinity-daemonset
+  namespace: kube-system
+spec:
+  ports:
+    - port: 443
+      targetPort: 8443
+  selector:
+    app: node-affinity-daemonset