From 6d411f5ae7a47bdd74723948c786e0dd9e2696e7 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Tue, 31 Oct 2023 10:56:08 -0500 Subject: [PATCH 001/114] Update orthanc-service.conf (#2386) --- kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf b/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf index 2eb77b1790..70d2918ea6 100644 --- a/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf @@ -10,6 +10,12 @@ location /orthanc/ { if ($request_uri ~ "^/orthanc/dicom-web/studies/") { set $authz_method "read"; set $authz_resource "/services/orthanc/studies"; + + if ($request_method = POST) { + return 403; + # set $authz_method "create"; + # set $authz_resource "/services/orthanc/studies"; + } } auth_request /gen3-authz; From ae02274ad52d09f95940b7757fdb0b78b7460d52 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Wed, 1 Nov 2023 09:15:02 -0500 Subject: [PATCH 002/114] Jenkins add "/usr/share/dict/words" (#2399) --- Docker/jenkins/Jenkins-CI-Worker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile index f0da68f696..242d5e74da 100644 --- a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile @@ -116,6 +116,9 @@ RUN curl -sS -o - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-ke && apt-get -y update \ && apt-get -y install google-chrome-stable +# data-simulator needs "/usr/share/dict/words" to generate data that isn't random strings +RUN apt-get install --reinstall wamerican + # update /etc/sudoers RUN sed 's/^%sudo/#%sudo/' /etc/sudoers > /etc/sudoers.bak \ && /bin/echo -e "\n%sudo ALL=(ALL:ALL) NOPASSWD:ALL\n" >> /etc/sudoers.bak \ From 9f35d535bdcc3ff1d29054ecd39caec9e3e11d88 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Wed, 1 Nov 2023 14:15:15 -0500 Subject: [PATCH 003/114] fix: allow only data commons origin (#2391) --- kube/services/revproxy/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/revproxy/nginx.conf b/kube/services/revproxy/nginx.conf index 2e3a3b1512..d0e14f49b6 100644 --- a/kube/services/revproxy/nginx.conf +++ b/kube/services/revproxy/nginx.conf @@ -236,7 +236,7 @@ server { # This overrides the individual services # set $allow_origin "*"; - if ($http_origin) { + if ($http_origin = "https://$host") { set $allow_origin "$http_origin"; } From 358773b6a091a1e985cfc98a85cc04c028a92104 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Thu, 2 Nov 2023 09:22:33 -0600 Subject: [PATCH 004/114] Update mutate-guppy-config-for-guppy-test.sh (#2403) --- gen3/bin/mutate-guppy-config-for-guppy-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen3/bin/mutate-guppy-config-for-guppy-test.sh b/gen3/bin/mutate-guppy-config-for-guppy-test.sh index de7da10d5c..151bb71697 100644 --- a/gen3/bin/mutate-guppy-config-for-guppy-test.sh +++ b/gen3/bin/mutate-guppy-config-for-guppy-test.sh @@ -16,7 +16,7 @@ sed -i 's/\(.*\)"index": "\(.*\)_etl",$/\1"index": "jenkins_subject_alias",/' or # for bloodpac-like envs sed -i 's/\(.*\)"index": "\(.*\)_case",$/\1"index": "jenkins_subject_alias",/' original_guppy_config.yaml # the pre-defined Canine index works with subject ONLY (never case) -sed -i 's/\(.*\)"type": "case"$/\1"type": "subject"/' original_guppy_config.yaml +# sed -i 's/\(.*\)"type": "case"$/\1"type": "subject"/' original_guppy_config.yaml sed -i 's/\(.*\)"index": "\(.*\)_file",$/\1"index": "jenkins_file_alias",/' original_guppy_config.yaml sed -i 's/\(.*\)"config_index": "\(.*\)_array-config",$/\1"config_index": "jenkins_configs_alias",/' original_guppy_config.yaml From c96599951a48edb50814db84789771ad94a205d9 Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Thu, 2 Nov 2023 17:22:24 +0100 Subject: [PATCH 005/114] Fix: set ARBORIST_URL using the kube-setup-ohdsi.sh (#2400) * fix: set ARBORIST_URL using the kube-setup-ohdsi.sh * fix: change arborist url to internal one * fix: get arborist_url directly from manifest.json --------- Co-authored-by: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> --- gen3/bin/kube-setup-ohdsi.sh | 2 ++ kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml | 7 ------- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/gen3/bin/kube-setup-ohdsi.sh b/gen3/bin/kube-setup-ohdsi.sh index d586570dba..14b35a7146 100644 --- a/gen3/bin/kube-setup-ohdsi.sh +++ b/gen3/bin/kube-setup-ohdsi.sh @@ -87,6 +87,8 @@ setup_secrets() { export DB_HOST=$(jq -r ".db_host" <<< "$dbcreds") export FENCE_URL="https://${hostname}/user/user" + # get arborist_url from manifest.json: + export ARBORIST_URL=$(g3k_manifest_lookup .global.arborist_url) export FENCE_METADATA_URL="https://${hostname}/.well-known/openid-configuration" export FENCE_CLIENT_ID=$(jq -r ".FENCE_CLIENT_ID" <<< "$appcreds") export FENCE_CLIENT_SECRET=$(jq -r ".FENCE_CLIENT_SECRET" <<< "$appcreds") diff --git a/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml b/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml index 258aa8f875..65d6ed38c9 100644 --- a/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml +++ b/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml @@ -59,13 +59,6 @@ spec: containers: - name: ohdsi-webapi GEN3_OHDSI-WEBAPI_IMAGE|-image: quay.io/cdis/ohdsi-webapi:latest-| - env: - - name: ARBORIST_URL - valueFrom: - configMapKeyRef: - name: manifest-global - key: arborist_url - optional: true livenessProbe: httpGet: path: /WebAPI/info/ From 997c596d86d22b4e085bc6da4e714050f06c7d16 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Thu, 2 Nov 2023 14:30:25 -0600 Subject: [PATCH 006/114] updating node monitor to include environment name (#2404) * updating node monitor to include environment name * moving the "!" --- kube/services/node-monitors/node-not-ready.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kube/services/node-monitors/node-not-ready.yaml b/kube/services/node-monitors/node-not-ready.yaml index 6626b55075..500832fc34 100644 --- a/kube/services/node-monitors/node-not-ready.yaml +++ b/kube/services/node-monitors/node-not-ready.yaml @@ -22,6 +22,11 @@ spec: configMapKeyRef: name: global key: slack_webhook + - name: ENVIRONMENT + valueFrom: + configMapKeyRef: + name: global + key: environment command: ["/bin/bash"] args: @@ -36,7 +41,7 @@ spec: echo "Nodes reporting 'NodeStatusNeverUpdated', sending an alert:" echo "$NODES" # Send alert to Slack - curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"WARNING: Node \`${NODES}\` is stuck in "NotReady"!\"}" $SLACK_WEBHOOK_URL + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"WARNING: Node \`${NODES}\` is stuck in "NotReady" in \`${ENVIRONMENT}\`! \"}" $SLACK_WEBHOOK_URL else echo "No nodes reporting 'NodeStatusNeverUpdated'" fi From 816b9831aefac6cd64e49d957eb8444d82c12d8e Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Mon, 6 Nov 2023 13:34:11 -0600 Subject: [PATCH 007/114] Update web_whitelist (#2405) --- files/squid_whitelist/web_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index 349d1e022f..063eab2e69 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -76,6 +76,7 @@ go.googlesource.com golang.org gopkg.in grafana.com +grafana.github.io http.us.debian.org ifconfig.io ingress.coralogix.us From c6aeccd8f96382d7d32389e762a3bc31e770d821 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Tue, 7 Nov 2023 09:43:49 -0600 Subject: [PATCH 008/114] feat: add app-config.js for ohif-viewer (#2406) --- gen3/bin/kube-setup-dicom.sh | 13 ++ kube/services/ohif-viewer/app-config.js | 209 ++++++++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 kube/services/ohif-viewer/app-config.js diff --git a/gen3/bin/kube-setup-dicom.sh b/gen3/bin/kube-setup-dicom.sh index 85114f33fe..42110eea2d 100644 --- a/gen3/bin/kube-setup-dicom.sh +++ b/gen3/bin/kube-setup-dicom.sh @@ -82,6 +82,19 @@ EOM } EOM fi + + if g3k_manifest_lookup .versions["dicom-server"] > /dev/null 2>&1; then + export DICOM_SERVER_URL="/dicom-server" + gen3_log_info "attaching ohif viewer to old dicom-server (orthanc w/ aurora)" + fi + + if g3k_manifest_lookup .versions["orthanc"] > /dev/null 2>&1; then + export DICOM_SERVER_URL="/orthanc" + gen3_log_info "attaching ohif viewer to new dicom-server (orthanc w/ s3)" + fi + + envsubst <"${GEN3_HOME}/kube/services/ohif-viewer/app-config.js" > "$secretsFolder/app-config.js" + gen3 secrets sync 'setup orthanc-s3-g3auto secrets' } diff --git a/kube/services/ohif-viewer/app-config.js b/kube/services/ohif-viewer/app-config.js new file mode 100644 index 0000000000..6768726f4c --- /dev/null +++ b/kube/services/ohif-viewer/app-config.js @@ -0,0 +1,209 @@ +window.config = { + routerBasename: '/ohif-viewer/', + // whiteLabeling: {}, + extensions: [], + modes: [], + customizationService: { + // Shows a custom route -access via http://localhost:3000/custom + // helloPage: '@ohif/extension-default.customizationModule.helloPage', + }, + showStudyList: true, + // some windows systems have issues with more than 3 web workers + maxNumberOfWebWorkers: 3, + // below flag is for performance reasons, but it might not work for all servers + omitQuotationForMultipartRequest: true, + showWarningMessageForCrossOrigin: true, + showCPUFallbackMessage: true, + showLoadingIndicator: true, + strictZSpacingForVolumeViewport: true, + maxNumRequests: { + interaction: 100, + thumbnail: 75, + // Prefetch number is dependent on the http protocol. For http 2 or + // above, the number of requests can be go a lot higher. + prefetch: 25, + }, + // filterQueryParam: false, + defaultDataSourceName: 'dicomweb', + /* Dynamic config allows user to pass "configUrl" query string this allows to load config without recompiling application. The regex will ensure valid configuration source */ + // dangerouslyUseDynamicConfig: { + // enabled: true, + // // regex will ensure valid configuration source and default is /.*/ which matches any character. To use this, setup your own regex to choose a specific source of configuration only. + // // Example 1, to allow numbers and letters in an absolute or sub-path only. + // // regex: /(0-9A-Za-z.]+)(\/[0-9A-Za-z.]+)*/ + // // Example 2, to restricts to either hosptial.com or othersite.com. + // // regex: /(https:\/\/hospital.com(\/[0-9A-Za-z.]+)*)|(https:\/\/othersite.com(\/[0-9A-Za-z.]+)*)/ + // regex: /.*/, + // }, + dataSources: [ + { + friendlyName: 'dcmjs DICOMWeb Server', + namespace: '@ohif/extension-default.dataSourcesModule.dicomweb', + sourceName: 'dicomweb', + configuration: { + name: 'dicomweb', + wadoUriRoot: '$DICOM_SERVER_URL/wado', + qidoRoot: '$DICOM_SERVER_URL/dicom-web', + wadoRoot: '$DICOM_SERVER_URL/dicom-web', + + qidoSupportsIncludeField: false, + supportsReject: false, + imageRendering: 'wadors', + thumbnailRendering: 'wadors', + enableStudyLazyLoad: true, + supportsFuzzyMatching: false, + supportsWildcard: true, + staticWado: true, + singlepart: 'bulkdata,video', + // whether the data source should use retrieveBulkData to grab metadata, + // and in case of relative path, what would it be relative to, options + // are in the series level or study level (some servers like series some study) + bulkDataURI: { + enabled: true, + relativeResolution: 'studies', + }, + }, + }, + { + friendlyName: 'dicomweb delegating proxy', + namespace: '@ohif/extension-default.dataSourcesModule.dicomwebproxy', + sourceName: 'dicomwebproxy', + configuration: { + name: 'dicomwebproxy', + }, + }, + { + friendlyName: 'dicom json', + namespace: '@ohif/extension-default.dataSourcesModule.dicomjson', + sourceName: 'dicomjson', + configuration: { + name: 'json', + }, + }, + { + friendlyName: 'dicom local', + namespace: '@ohif/extension-default.dataSourcesModule.dicomlocal', + sourceName: 'dicomlocal', + configuration: {}, + }, + ], + httpErrorHandler: error => { + // This is 429 when rejected from the public idc sandbox too often. + console.warn(error.status); + + // Could use services manager here to bring up a dialog/modal if needed. + console.warn('test, navigate to https://ohif.org/'); + }, + // whiteLabeling: { + // /* Optional: Should return a React component to be rendered in the "Logo" section of the application's Top Navigation bar */ + // createLogoComponentFn: function (React) { + // return React.createElement( + // 'a', + // { + // target: '_self', + // rel: 'noopener noreferrer', + // className: 'text-purple-600 line-through', + // href: '/', + // }, + // React.createElement('img', + // { + // src: './assets/customLogo.svg', + // className: 'w-8 h-8', + // } + // )) + // }, + // }, + hotkeys: [ + { + commandName: 'incrementActiveViewport', + label: 'Next Viewport', + keys: ['right'], + }, + { + commandName: 'decrementActiveViewport', + label: 'Previous Viewport', + keys: ['left'], + }, + { commandName: 'rotateViewportCW', label: 'Rotate Right', keys: ['r'] }, + { commandName: 'rotateViewportCCW', label: 'Rotate Left', keys: ['l'] }, + { commandName: 'invertViewport', label: 'Invert', keys: ['i'] }, + { + commandName: 'flipViewportHorizontal', + label: 'Flip Horizontally', + keys: ['h'], + }, + { + commandName: 'flipViewportVertical', + label: 'Flip Vertically', + keys: ['v'], + }, + { commandName: 'scaleUpViewport', label: 'Zoom In', keys: ['+'] }, + { commandName: 'scaleDownViewport', label: 'Zoom Out', keys: ['-'] }, + { commandName: 'fitViewportToWindow', label: 'Zoom to Fit', keys: ['='] }, + { commandName: 'resetViewport', label: 'Reset', keys: ['space'] }, + { commandName: 'nextImage', label: 'Next Image', keys: ['down'] }, + { commandName: 'previousImage', label: 'Previous Image', keys: ['up'] }, + // { + // commandName: 'previousViewportDisplaySet', + // label: 'Previous Series', + // keys: ['pagedown'], + // }, + // { + // commandName: 'nextViewportDisplaySet', + // label: 'Next Series', + // keys: ['pageup'], + // }, + { + commandName: 'setToolActive', + commandOptions: { toolName: 'Zoom' }, + label: 'Zoom', + keys: ['z'], + }, + // ~ Window level presets + { + commandName: 'windowLevelPreset1', + label: 'W/L Preset 1', + keys: ['1'], + }, + { + commandName: 'windowLevelPreset2', + label: 'W/L Preset 2', + keys: ['2'], + }, + { + commandName: 'windowLevelPreset3', + label: 'W/L Preset 3', + keys: ['3'], + }, + { + commandName: 'windowLevelPreset4', + label: 'W/L Preset 4', + keys: ['4'], + }, + { + commandName: 'windowLevelPreset5', + label: 'W/L Preset 5', + keys: ['5'], + }, + { + commandName: 'windowLevelPreset6', + label: 'W/L Preset 6', + keys: ['6'], + }, + { + commandName: 'windowLevelPreset7', + label: 'W/L Preset 7', + keys: ['7'], + }, + { + commandName: 'windowLevelPreset8', + label: 'W/L Preset 8', + keys: ['8'], + }, + { + commandName: 'windowLevelPreset9', + label: 'W/L Preset 9', + keys: ['9'], + }, + ], +}; From e1e2fdeeac9d9baafc3841f8aca248b0ad58ad35 Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:47:28 -0600 Subject: [PATCH 009/114] Update orthanc-service.conf (#2402) --- .../gen3.nginx.conf/orthanc-service.conf | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf b/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf index 70d2918ea6..ed736189c4 100644 --- a/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/orthanc-service.conf @@ -7,17 +7,6 @@ location /orthanc/ { set $authz_method "create"; set $authz_service "orthanc"; - if ($request_uri ~ "^/orthanc/dicom-web/studies/") { - set $authz_method "read"; - set $authz_resource "/services/orthanc/studies"; - - if ($request_method = POST) { - return 403; - # set $authz_method "create"; - # set $authz_resource "/services/orthanc/studies"; - } - } - auth_request /gen3-authz; proxy_set_header Authorization "Basic cHVibGljOmhlbGxv"; @@ -30,3 +19,23 @@ location /orthanc/ { # no limit to payload size so we can upload large DICOM files client_max_body_size 0; } + +location /orthanc/dicom-web/studies/ { + set $authz_method "read"; + set $authz_resource "/services/orthanc/studies"; + set $authz_service "orthanc"; + + auth_request /gen3-authz; + if ($request_method = POST) { + return 403; + } + proxy_set_header Authorization "Basic cHVibGljOmhlbGxv"; + + set $proxy_service "orthanc"; + set $upstream http://orthanc-service.$namespace.svc.cluster.local; + rewrite ^/orthanc/(.*) /$1 break; + proxy_pass $upstream; + + # no limit to payload size so we can upload large DICOM files + client_max_body_size 0; +} From 7673146280cb3a53d7803ca064becfacf9cc3f86 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Wed, 8 Nov 2023 14:22:45 -0600 Subject: [PATCH 010/114] Update ohif-viewer-service.conf (#2410) --- .../revproxy/gen3.nginx.conf/ohif-viewer-service.conf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kube/services/revproxy/gen3.nginx.conf/ohif-viewer-service.conf b/kube/services/revproxy/gen3.nginx.conf/ohif-viewer-service.conf index 9a20bc832f..22926bcf05 100644 --- a/kube/services/revproxy/gen3.nginx.conf/ohif-viewer-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/ohif-viewer-service.conf @@ -3,11 +3,12 @@ location /ohif-viewer/ { # return 403 "failed csrf check"; # } - set $authz_resource "/services/ohif-viewer"; - set $authz_method "read"; - set $authz_service "ohif-viewer"; + # see if this can be fixed in the future for anonymous access + # set $authz_resource "/services/ohif-viewer"; + # set $authz_method "read"; + # set $authz_service "ohif-viewer"; - auth_request /gen3-authz; + # auth_request /gen3-authz; set $proxy_service "ohif-viewer"; set $upstream http://ohif-viewer-service.$namespace.svc.cluster.local; From fee95411e38ff3fb17ed08d164929f2f3c73037e Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 9 Nov 2023 09:43:12 -0600 Subject: [PATCH 011/114] Add log when gitops-sync initially detects changes (#2411) --- gen3/bin/gitops.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gen3/bin/gitops.sh b/gen3/bin/gitops.sh index fda6d4ffa9..bc0358499d 100644 --- a/gen3/bin/gitops.sh +++ b/gen3/bin/gitops.sh @@ -445,8 +445,13 @@ gen3_gitops_sync() { echo "DRYRUN flag detected, not rolling" gen3_log_info "dict_roll: $dict_roll; versions_roll: $versions_roll; portal_roll: $portal_roll; etl_roll: $etl_roll; fence_roll: $fence_roll" else - if [[ ( "$dict_roll" = true ) || ( "$versions_roll" = true ) || ( "$portal_roll" = true )|| ( "$etl_roll" = true ) || ( "$covid_cronjob_roll" = true ) || ("fence_roll" = true) ]]; then + if [[ ( "$dict_roll" = true ) || ( "$versions_roll" = true ) || ( "$portal_roll" = true )|| ( "$etl_roll" = true ) || ( "$covid_cronjob_roll" = true ) || ("$fence_roll" = true) ]]; then echo "changes detected, rolling" + tmpHostname=$(gen3 api hostname) + if [[ $slack = true ]]; then + curl -X POST --data-urlencode "payload={\"text\": \"Gitops-sync Cron: Changes detected on ${tmpHostname} - rolling...\"}" "${slackWebHook}" + fi + # run etl job before roll all so guppy can pick up changes if [[ "$etl_roll" = true ]]; then gen3 update_config etl-mapping "$(gen3 gitops folder)/etlMapping.yaml" @@ -472,7 +477,6 @@ gen3_gitops_sync() { rollRes=$? # send result to slack if [[ $slack = true ]]; then - tmpHostname=$(gen3 api hostname) resStr="SUCCESS" color="#1FFF00" if [[ $rollRes != 0 ]]; then From e51eba5f3eb65a3bf144b39ac5e95934d5f08d5f Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:25:32 -0600 Subject: [PATCH 012/114] Add rmq.n3c.ncats.io to Squid whitelist (#2413) --- files/squid_whitelist/web_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index 063eab2e69..0cbc0a6bc8 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -145,6 +145,7 @@ repos.sensuapp.org repo.vmware.com repository.cloudera.com resource.metadatacenter.org +rmq.n3c.ncats.io rules.emergingthreats.net rweb.quant.ku.edu sa-update.dnswl.org From f7d0469b087a4020aa5cf3c9aab52de00e54a4bc Mon Sep 17 00:00:00 2001 From: George Thomas <98996322+george42-ctds@users.noreply.github.com> Date: Wed, 15 Nov 2023 07:27:12 -0800 Subject: [PATCH 013/114] Chore/create cedar client (#2407) * (chore HP-1273): add CEDAR client creation * (fix PPS-655): change comment * (fix PPS-655): remove last comma in json secret * (chore HP-1273): if creation fails then don't start wrapper service * (chore HP-1273): don't echo secrets on client creation failure --- gen3/bin/kube-setup-cedar-wrapper.sh | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/gen3/bin/kube-setup-cedar-wrapper.sh b/gen3/bin/kube-setup-cedar-wrapper.sh index 9a899a7700..c8f0d03c6c 100644 --- a/gen3/bin/kube-setup-cedar-wrapper.sh +++ b/gen3/bin/kube-setup-cedar-wrapper.sh @@ -1,6 +1,58 @@ source "${GEN3_HOME}/gen3/lib/utils.sh" gen3_load "gen3/lib/kube-setup-init" +create_client_and_secret() { + local hostname=$(gen3 api hostname) + local client_name="cedar_ingest_client" + gen3_log_info "kube-setup-cedar-wrapper" "creating fence ${client_name} for $hostname" + # delete any existing fence cedar clients + g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client ${client_name} > /dev/null 2>&1 + local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client ${client_name} --grant-types client_credentials | tail -1) + # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET') + if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then + gen3_log_err "kube-setup-cedar-wrapper" "Failed generating ${client_name}" + return 1 + else + local client_id="${BASH_REMATCH[2]}" + local client_secret="${BASH_REMATCH[3]}" + gen3_log_info "Create cedar-client secrets file" + cat - < /dev/null 2>&1; then + local have_cedar_client_secret="1" + else + gen3_log_info "No g3auto cedar-client key present in secret" + fi + + local client_name="cedar_ingest_client" + local client_list=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-list) + local client_count=$(echo "$client_list=" | grep -cE "'name':.*'${client_name}'") + gen3_log_info "CEDAR client count = ${client_count}" + + if [[ -z $have_cedar_client_secret ]] || [[ ${client_count} -lt 1 ]]; then + gen3_log_info "Creating new cedar-ingest client and secret" + local credsPath="$(gen3_secrets_folder)/g3auto/cedar/${cedar_creds_file}" + if ! create_client_and_secret > $credsPath; then + gen3_log_err "Failed to setup cedar-ingest secret" + return 1 + else + gen3 secrets sync + gen3 job run usersync + fi + fi +} + [[ -z "$GEN3_ROLL_ALL" ]] && gen3 kube-setup-secrets if ! g3kubectl get secrets/cedar-g3auto > /dev/null 2>&1; then @@ -8,6 +60,9 @@ if ! g3kubectl get secrets/cedar-g3auto > /dev/null 2>&1; then return 1 fi +gen3_log_info "Checking cedar-client creds" +setup_creds + if ! gen3 secrets decode cedar-g3auto cedar_api_key.txt > /dev/null 2>&1; then gen3_log_err "No CEDAR api key present in cedar-g3auto secret, not rolling CEDAR wrapper" return 1 From 75c7ec815a848e21136cb9daf533a84186ddb4f2 Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Thu, 16 Nov 2023 09:25:48 -0800 Subject: [PATCH 014/114] Quarantine jenkins-dcp until migration to ES7 (#2417) --- files/scripts/ci-env-pool-reset.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/files/scripts/ci-env-pool-reset.sh b/files/scripts/ci-env-pool-reset.sh index 362cfbfd57..c0c1f67c6d 100644 --- a/files/scripts/ci-env-pool-reset.sh +++ b/files/scripts/ci-env-pool-reset.sh @@ -29,7 +29,6 @@ source "${GEN3_HOME}/gen3/gen3setup.sh" cat - > jenkins-envs-services.txt < Date: Fri, 17 Nov 2023 16:19:28 -0500 Subject: [PATCH 015/114] Chore/add jenkins2 ci worker service (#2418) * adding jenkins2-ci-worker service * renamed the file --- .../jenkins2-agent-service.yaml | 17 ++ .../jenkins2-ci-worker-deploy.yaml | 149 ++++++++++++++++++ .../jenkins2-ci-worker-pvc.yaml | 12 ++ 3 files changed, 178 insertions(+) create mode 100644 kube/services/jenkins2-ci-worker/jenkins2-agent-service.yaml create mode 100644 kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml create mode 100644 kube/services/jenkins2-ci-worker/jenkins2-ci-worker-pvc.yaml diff --git a/kube/services/jenkins2-ci-worker/jenkins2-agent-service.yaml b/kube/services/jenkins2-ci-worker/jenkins2-agent-service.yaml new file mode 100644 index 0000000000..7f4e58109c --- /dev/null +++ b/kube/services/jenkins2-ci-worker/jenkins2-agent-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: jenkins-agent-service + name: jenkins-agent + namespace: default +spec: + ports: + - name: slavelistener + port: 50000 + protocol: TCP + targetPort: 50000 + selector: + app: jenkins + sessionAffinity: None + type: ClusterIP diff --git a/kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml b/kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml new file mode 100644 index 0000000000..3dea38a5cb --- /dev/null +++ b/kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml @@ -0,0 +1,149 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jenkins-ci-worker-deployment +spec: + selector: + # Only select pods based on the 'app' label + matchLabels: + app: jenkins-ci-worker + template: + metadata: + labels: + app: jenkins-ci-worker + # for network policy + netnolimit: "yes" + annotations: + "cluster-autoscaler.kubernetes.io/safe-to-evict": "false" + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/capacityType + operator: In + values: + - ONDEMAND + - matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + serviceAccountName: jenkins-service + securityContext: + runAsUser: 1000 + fsGroup: 1000 + initContainers: + - args: + - -c + - | + # fix permissions for /var/run/docker.sock + chmod 666 /var/run/docker.sock + echo "done" + command: + - /bin/bash + image: quay.io/cdis/awshelper:master + imagePullPolicy: Always + name: awshelper + resources: {} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/run/docker.sock + name: dockersock + containers: + # + # See for details on running docker in a pod: + # https://estl.tech/accessing-docker-from-a-kubernetes-pod-68996709c04b + # + - name: jenkins-worker + image: "quay.io/cdis/gen3-ci-worker:master" + ports: + - containerPort: 8080 + env: + - name: JENKINS_URL + value: "https://jenkins2.planx-pla.net" + - name: JENKINS_SECRET + valueFrom: + secretKeyRef: + name: jenkins-ci-worker-g3auto + key: jenkins-jnlp-agent-secret + - name: JENKINS_AGENT_NAME + value: "gen3-ci-worker" + - name: JENKINS_TUNNEL + value: "jenkins-agent:50000" + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: JAVA_OPTS + value: "-Xmx3072m" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: jenkins-secret + key: aws_access_key_id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: jenkins-secret + key: aws_secret_access_key + - name: GOOGLE_EMAIL_AUX1 + valueFrom: + secretKeyRef: + name: google-acct1 + key: email + - name: GOOGLE_PASSWORD_AUX1 + valueFrom: + secretKeyRef: + name: google-acct1 + key: password + - name: GOOGLE_EMAIL_AUX2 + valueFrom: + secretKeyRef: + name: google-acct2 + key: email + - name: GOOGLE_PASSWORD_AUX2 + valueFrom: + secretKeyRef: + name: google-acct2 + key: password + - name: GOOGLE_APP_CREDS_JSON + valueFrom: + secretKeyRef: + name: jenkins-g3auto + key: google_app_creds.json + resources: + limits: + cpu: 0.9 + memory: 4096Mi + ephemeral-storage: 500Mi + imagePullPolicy: Always + volumeMounts: + - name: "cert-volume" + readOnly: true + mountPath: "/mnt/ssl/service.crt" + subPath: "service.crt" + - name: "cert-volume" + readOnly: true + mountPath: "/mnt/ssl/service.key" + subPath: "service.key" + - name: "ca-volume" + readOnly: true + mountPath: "/usr/local/share/ca-certificates/cdis/cdis-ca.crt" + subPath: "ca.pem" + - name: dockersock + mountPath: "/var/run/docker.sock" + imagePullPolicy: Always + volumes: + - name: cert-volume + secret: + secretName: "cert-jenkins-service" + - name: ca-volume + secret: + secretName: "service-ca" + - name: dockersock + hostPath: + path: /var/run/docker.sock diff --git a/kube/services/jenkins2-ci-worker/jenkins2-ci-worker-pvc.yaml b/kube/services/jenkins2-ci-worker/jenkins2-ci-worker-pvc.yaml new file mode 100644 index 0000000000..047e4e966e --- /dev/null +++ b/kube/services/jenkins2-ci-worker/jenkins2-ci-worker-pvc.yaml @@ -0,0 +1,12 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: datadir-jenkins-ci + annotations: + volume.beta.kubernetes.io/storage-class: gp2 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi From 421b762dd8452303e591416ca5ef749206189948 Mon Sep 17 00:00:00 2001 From: Atharva Rane <41084525+atharvar28@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:09:28 -0500 Subject: [PATCH 016/114] add kube-setup-jenkins2 script (#2419) * add kube-setup-jenkins2 script * change the env image name --- gen3/bin/kube-setup-jenkins2.sh | 71 +++++++++++++++++++++ kube/services/jenkins2/jenkins2-deploy.yaml | 2 +- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 gen3/bin/kube-setup-jenkins2.sh diff --git a/gen3/bin/kube-setup-jenkins2.sh b/gen3/bin/kube-setup-jenkins2.sh new file mode 100644 index 0000000000..f5233f978c --- /dev/null +++ b/gen3/bin/kube-setup-jenkins2.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Just a little helper for deploying jenkins onto k8s the first time +# + +set -e + +export WORKSPACE="${WORKSPACE:-$HOME}" +source "${GEN3_HOME}/gen3/lib/utils.sh" +gen3_load "gen3/gen3setup" + +gen3 kube-setup-secrets + +# +# Assume Jenkins should use 'jenkins' profile credentials in "${WORKSPACE}"/.aws/credentials +# +aws_access_key_id="$(aws configure get jenkins.aws_access_key_id)" +aws_secret_access_key="$(aws configure get jenkins.aws_secret_access_key)" +google_acct1_email="$(jq -r '.jenkins.google_acct1.email' < $(gen3_secrets_folder)/creds.json)" +google_acct1_password="$(jq -r '.jenkins.google_acct1.password' < $(gen3_secrets_folder)/creds.json)" +google_acct2_email="$(jq -r '.jenkins.google_acct2.email' < $(gen3_secrets_folder)/creds.json)" +google_acct2_password="$(jq -r '.jenkins.google_acct2.password' < $(gen3_secrets_folder)/creds.json)" + +if [ -z "$aws_access_key_id" -o -z "$aws_secret_access_key" ]; then + gen3_log_err 'not configuring jenkins - could not extract secrets from aws configure' + exit 1 +fi +if [[ -z "$google_acct1_email" || -z "$google_acct1_password" || -z "$google_acct2_email" || -z "$google_acct2_password" ]]; then + gen3_log_err "missing google credentials in '.jenkins' of creds.json" + exit 1 +fi + +if ! g3kubectl get secrets jenkins-secret > /dev/null 2>&1; then + # make it easy to rerun kube-setup-jenkins.sh + g3kubectl create secret generic jenkins-secret "--from-literal=aws_access_key_id=$aws_access_key_id" "--from-literal=aws_secret_access_key=$aws_secret_access_key" +fi +if ! g3kubectl get secrets google-acct1 > /dev/null 2>&1; then + g3kubectl create secret generic google-acct1 "--from-literal=email=${google_acct1_email}" "--from-literal=password=${google_acct1_password}" +fi +if ! g3kubectl get secrets google-acct2 > /dev/null 2>&1; then + g3kubectl create secret generic google-acct2 "--from-literal=email=${google_acct2_email}" "--from-literal=password=${google_acct2_password}" +fi + +if ! g3kubectl get storageclass gp2 > /dev/null 2>&1; then + g3kubectl apply -f "${GEN3_HOME}/kube/services/jenkins/10storageclass.yaml" +fi +if ! g3kubectl get persistentvolumeclaim datadir-jenkins > /dev/null 2>&1; then + g3kubectl apply -f "${GEN3_HOME}/kube/services/jenkins/00pvc.yaml" +fi + +# Note: jenkins service account is configured by `kube-setup-roles` +gen3 kube-setup-roles +# Note: only the 'default' namespace jenkins-service account gets a cluster rolebinding +g3kubectl apply -f "${GEN3_HOME}/kube/services/jenkins/clusterrolebinding-devops.yaml" + +# Note: requires Jenkins entry in cdis-manifest +gen3 roll jenkins2 +gen3 roll jenkins2-worker +gen3 roll jenkins2-ci-worker + +# +# Get the ARN of the SSL certificate for the commons - +# We'll optimistically assume it's a wildcard cert that +# is appropriate to also attach to the jenkins ELB +# +export ARN=$(g3kubectl get configmap global --output=jsonpath='{.data.revproxy_arn}') +if [[ ! -z $ARN ]]; then + envsubst <"${GEN3_HOME}/kube/services/jenkins/jenkins-service.yaml" | g3kubectl apply -f - +else + gen3_log_info "Global configmap not configured - not launching service (require SSL cert ARN)" +fi diff --git a/kube/services/jenkins2/jenkins2-deploy.yaml b/kube/services/jenkins2/jenkins2-deploy.yaml index ee838bae60..08365f811b 100644 --- a/kube/services/jenkins2/jenkins2-deploy.yaml +++ b/kube/services/jenkins2/jenkins2-deploy.yaml @@ -48,7 +48,7 @@ spec: # https://estl.tech/accessing-docker-from-a-kubernetes-pod-68996709c04b # - name: jenkins - GEN3_JENKINS_IMAGE + GEN3_JENKINS2_IMAGE ports: - containerPort: 8080 name: http From c9803f5bc310e84c87582c1f7c47783e256e784b Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:07:33 -0700 Subject: [PATCH 017/114] changing expected results to match sheepdog and fence-deploy.yaml changes (#2421) --- gen3/lib/testData/default/expectedFenceResult.yaml | 8 ++++++++ gen3/lib/testData/default/expectedSheepdogResult.yaml | 10 +++++++++- .../test1.manifest.g3k/expectedFenceResult.yaml | 7 +++++++ .../test1.manifest.g3k/expectedSheepdogResult.yaml | 7 +++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/gen3/lib/testData/default/expectedFenceResult.yaml b/gen3/lib/testData/default/expectedFenceResult.yaml index f6d76d790f..98c3605311 100644 --- a/gen3/lib/testData/default/expectedFenceResult.yaml +++ b/gen3/lib/testData/default/expectedFenceResult.yaml @@ -44,6 +44,13 @@ spec: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 + preference: + matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - weight: 99 preference: matchExpressions: - key: eks.amazonaws.com/capacityType @@ -136,6 +143,7 @@ spec: ports: - containerPort: 80 - containerPort: 443 + - containerPort: 6567 volumeMounts: # ----------------------------------------------------------------------------- # DEPRECATED! Remove when all commons are no longer using local_settings.py diff --git a/gen3/lib/testData/default/expectedSheepdogResult.yaml b/gen3/lib/testData/default/expectedSheepdogResult.yaml index b9db85a36c..a2bd3efcc4 100644 --- a/gen3/lib/testData/default/expectedSheepdogResult.yaml +++ b/gen3/lib/testData/default/expectedSheepdogResult.yaml @@ -17,6 +17,7 @@ spec: template: metadata: labels: + netnolimit: "yes" app: sheepdog release: production public: "yes" @@ -39,12 +40,19 @@ spec: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 + preference: + matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - weight: 99 preference: matchExpressions: - key: eks.amazonaws.com/capacityType operator: In values: - - ONDEMAND + - SPOT automountServiceAccountToken: false volumes: - name: config-volume diff --git a/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml b/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml index d4196c0708..adc35ad2f1 100644 --- a/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml +++ b/gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml @@ -47,6 +47,13 @@ spec: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 + preference: + matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - weight: 99 preference: matchExpressions: - key: eks.amazonaws.com/capacityType diff --git a/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml b/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml index f54fd3e03b..08407ae52b 100644 --- a/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml +++ b/gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml @@ -43,6 +43,13 @@ spec: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 + preference: + matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - weight: 99 preference: matchExpressions: - key: eks.amazonaws.com/capacityType From 41c61c2648db6ec1f373c5c93c09f4ac3d615005 Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Fri, 1 Dec 2023 09:30:15 -0800 Subject: [PATCH 018/114] Update jenkins versions (#2422) * Update Jenkins * Update Dockerfile * Update Dockerfile --- Docker/jenkins/Jenkins-CI-Worker/Dockerfile | 2 +- Docker/jenkins/Jenkins-Worker/Dockerfile | 2 +- Docker/jenkins/Jenkins/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile index 242d5e74da..6eeb8f4fd6 100644 --- a/Docker/jenkins/Jenkins-CI-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-CI-Worker/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/inbound-agent:jdk11 +FROM jenkins/inbound-agent:jdk21 USER root diff --git a/Docker/jenkins/Jenkins-Worker/Dockerfile b/Docker/jenkins/Jenkins-Worker/Dockerfile index c824690def..fec6b3203c 100644 --- a/Docker/jenkins/Jenkins-Worker/Dockerfile +++ b/Docker/jenkins/Jenkins-Worker/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/inbound-agent:jdk11 +FROM jenkins/inbound-agent:jdk21 USER root diff --git a/Docker/jenkins/Jenkins/Dockerfile b/Docker/jenkins/Jenkins/Dockerfile index ae39ac5740..8b8c58406b 100644 --- a/Docker/jenkins/Jenkins/Dockerfile +++ b/Docker/jenkins/Jenkins/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.415-jdk11 +FROM jenkins/jenkins:2.434-jdk21 USER root From 93cd549a851a95697b00689bac240ec7463b1cb9 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:17:33 -0500 Subject: [PATCH 019/114] Added a cronjob to reconcile karpenter resources, in case they don't get created by argo-events (#2426) * Setting up the Karpenter reconciler cronjob * Matching both configmaps * Putting configmap in right namespace * Fixing oopsie * Fixing namespaces * Duh * Removing dependency on Gen3, since that was a pain * Fixing name and adding auth * We need the label for network policies * This should work * Update application.yaml * Moving everything over to argo-events namespace * Removed configmap.yaml, and moved everything over to argo-events namespace --- .../argo-events/workflows/configmap.yaml | 25 ++++++- .../karpenter-reconciler/application.yaml | 22 ++++++ kube/services/karpenter-reconciler/auth.yaml | 44 ++++++++++++ .../karpenter-reconciler-cronjob.yaml | 71 +++++++++++++++++++ 4 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 kube/services/karpenter-reconciler/application.yaml create mode 100644 kube/services/karpenter-reconciler/auth.yaml create mode 100644 kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml diff --git a/kube/services/argo-events/workflows/configmap.yaml b/kube/services/argo-events/workflows/configmap.yaml index eb5f1b04f4..9fc4b78268 100644 --- a/kube/services/argo-events/workflows/configmap.yaml +++ b/kube/services/argo-events/workflows/configmap.yaml @@ -21,9 +21,28 @@ data: - key: karpenter.k8s.aws/instance-family operator: In values: - - c6i - - c7i - - m7i + - c6a.large + - c6a.xlarge + - c6a.2xlarge + - c6a.4xlarge + - c6a.8xlarge + - c6a.12xlarge + - c6i.large + - c6i.xlarge + - c6i.2xlarge + - c6i.4xlarge + - c6i.8xlarge + - c6i.12xlarge + - m6a.2xlarge + - m6a.4xlarge + - m6a.8xlarge + - m6a.12xlarge + - m6a.16xlarge + - m6i.2xlarge + - m6i.4xlarge + - m6i.8xlarge + - m6i.12xlarge + - m6i.16xlarge taints: - key: role value: $WORKFLOW_NAME diff --git a/kube/services/karpenter-reconciler/application.yaml b/kube/services/karpenter-reconciler/application.yaml new file mode 100644 index 0000000000..c2d945b475 --- /dev/null +++ b/kube/services/karpenter-reconciler/application.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: karpenter-reconciler-application + namespace: argocd +spec: + destination: + namespace: kube-system + server: https://kubernetes.argo-events.svc + project: argo-events + source: + repoURL: https://github.com/uc-cdis/cloud-automation.git + targetRevision: master + path: kube/services/karpenter-reconciler + directory: + exclude: "application.yaml" + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/kube/services/karpenter-reconciler/auth.yaml b/kube/services/karpenter-reconciler/auth.yaml new file mode 100644 index 0000000000..c159028ab3 --- /dev/null +++ b/kube/services/karpenter-reconciler/auth.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: karpenter-reconciler + namespace: argo-events +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: karpenter-admin-binding-reconciler +subjects: + - kind: ServiceAccount + name: karpenter-reconciler + namespace: argo-events +roleRef: + kind: ClusterRole + name: karpenter-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: workflow-viewer-reconciler +subjects: + - kind: ServiceAccount + name: karpenter-reconciler + namespace: argo-events +roleRef: + kind: ClusterRole + name: argo-argo-workflows-view + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: viewer-reconciler +subjects: + - kind: ServiceAccount + name: karpenter-reconciler + namespace: argo-events +roleRef: + kind: ClusterRole + name: system:aggregate-to-view + apiGroup: rbac.authorization.k8s.io diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml new file mode 100644 index 0000000000..c5b5019186 --- /dev/null +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -0,0 +1,71 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: karpenter-reconciler-cronjob + namespace: argo-events +spec: + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: gen3job + spec: + serviceAccount: karpenter-reconciler + volumes: + - name: karpenter-templates-volume + configMap: + name: karpenter-templates + containers: + - name: karpenter-reconciler + image: quay.io/cdis/awshelper + volumeMounts: + - name: karpenter-templates-volume + mountPath: /manifests + env: + - name: PROVISIONER_TEMPLATE + value: /manifests/provisioner.yaml + - name: AWSNODETEMPLATE_TEMPLATE + value: /manifests/nodetemplate.yaml + command: ["/bin/bash"] + args: + - "-c" + - | + #!/bin/bash + if [ -z "$PROVISIONER_TEMPLATE" ]; then + PROVISIONER_TEMPLATE="provisioner.yaml" + fi + + if [ -z "$AWSNODETEMPLATE_TEMPLATE" ]; then + AWSNODETEMPLATE_TEMPLATE="nodetemplate.yaml" + fi + + ENVIRONMENT=$(kubectl get configmap global -o jsonpath="{.data.environment}") + + RAW_WORKFLOWS=$(kubectl get workflows -n argo -o yaml) + + WORKFLOWS=$(echo "${RAW_WORKFLOWS}" | yq -r '.items[] | [.metadata.name, .metadata.labels.gen3username] | join(" ")') + + WORKFLOW_ARRAY=() + + while IFS= read -r line; do + WORKFLOW_ARRAY+=("$line") + done <<< "$WORKFLOWS" + + for workflow in "${WORKFLOW_ARRAY[@]}" + do + workflow_name=$(echo "$workflow" | awk '{print $1}') + workflow_user=$(echo "$workflow" | awk '{print $2}') + + if ! kubectl get awsnodetemplate workflow-$workflow_name >/dev/null 2>&1; then + echo "No awsnodetemplate found for ${workflow_name}, creating one" + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USER_NAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f - + fi + + if ! kubectl get provisioner workflow-$workflow_name >/dev/null 2>&1; then + echo "No provisioner found for ${workflow_name}, creating one" + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USER_NAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f - + fi + done + restartPolicy: OnFailure From 49051c01b73ebb30aa947c8d868dfe73916697af Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:48:01 -0500 Subject: [PATCH 020/114] Update application.yaml (#2427) --- kube/services/karpenter-reconciler/application.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kube/services/karpenter-reconciler/application.yaml b/kube/services/karpenter-reconciler/application.yaml index c2d945b475..fb0fab8711 100644 --- a/kube/services/karpenter-reconciler/application.yaml +++ b/kube/services/karpenter-reconciler/application.yaml @@ -6,8 +6,8 @@ metadata: spec: destination: namespace: kube-system - server: https://kubernetes.argo-events.svc - project: argo-events + server: https://kubernetes.default.svc + project: default source: repoURL: https://github.com/uc-cdis/cloud-automation.git targetRevision: master From 59d3e11d2aabe0f10f5daff8eade21c8fb839523 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:15:53 -0500 Subject: [PATCH 021/114] Feat/karpenter reconciler (#2428) * Setting up the Karpenter reconciler cronjob * Matching both configmaps * Putting configmap in right namespace * Fixing oopsie * Fixing namespaces * Duh * Removing dependency on Gen3, since that was a pain * Fixing name and adding auth * We need the label for network policies * This should work * Update application.yaml * Moving everything over to argo-events namespace * Removed configmap.yaml, and moved everything over to argo-events namespace * Fixing stuff * Fixing it so they can work together * Fixing a typo in the configmap * Username, not "user name" --- .../argo-events/workflows/configmap.yaml | 26 +++++++++---------- .../argo-events/workflows/sensor-created.yaml | 4 +-- .../karpenter-reconciler-cronjob.yaml | 7 ++--- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/kube/services/argo-events/workflows/configmap.yaml b/kube/services/argo-events/workflows/configmap.yaml index 9fc4b78268..ae1c16653c 100644 --- a/kube/services/argo-events/workflows/configmap.yaml +++ b/kube/services/argo-events/workflows/configmap.yaml @@ -8,7 +8,7 @@ data: apiVersion: karpenter.sh/v1alpha5 kind: Provisioner metadata: - name: workflow-$WORKFLOW_NAME + name: workflow-WORKFLOW_NAME spec: requirements: - key: karpenter.sh/capacity-type @@ -18,7 +18,7 @@ data: operator: In values: - amd64 - - key: karpenter.k8s.aws/instance-family + - key: node.kubernetes.io/instance-type operator: In values: - c6a.large @@ -45,15 +45,15 @@ data: - m6i.16xlarge taints: - key: role - value: $WORKFLOW_NAME + value: WORKFLOW_NAME effect: NoSchedule labels: - role: $WORKFLOW_NAME + role: WORKFLOW_NAME limits: resources: cpu: 2000 providerRef: - name: workflow-$WORKFLOW_NAME + name: workflow-WORKFLOW_NAME # Kill nodes after 30 days to ensure they stay up to date ttlSecondsUntilExpired: 2592000 ttlSecondsAfterEmpty: 10 @@ -62,18 +62,18 @@ data: apiVersion: karpenter.k8s.aws/v1alpha1 kind: AWSNodeTemplate metadata: - name: workflow-$WORKFLOW_NAME + name: workflow-WORKFLOW_NAME spec: subnetSelector: - karpenter.sh/discovery: $ENVIRONMENT + karpenter.sh/discovery: ENVIRONMENT securityGroupSelector: - karpenter.sh/discovery: $ENVIRONMENT-workflow + karpenter.sh/discovery: ENVIRONMENT-workflow tags: - Environment: $ENVIRONMENT - Name: eks-$ENVIRONMENT-workflow-karpenter - karpenter.sh/discovery: $ENVIRONMENT - workflowname: $WORKFLOW_NAME - gen3username: $GEN3_USERNAME + Environment: ENVIRONMENT + Name: eks-ENVIRONMENT-workflow-karpenter + karpenter.sh/discovery: ENVIRONMENT + workflowname: WORKFLOW_NAME + gen3username: GEN3_USERNAME gen3service: argo-workflows purpose: workflow metadataOptions: diff --git a/kube/services/argo-events/workflows/sensor-created.yaml b/kube/services/argo-events/workflows/sensor-created.yaml index 7b1b9d62fe..4221f57423 100644 --- a/kube/services/argo-events/workflows/sensor-created.yaml +++ b/kube/services/argo-events/workflows/sensor-created.yaml @@ -60,11 +60,11 @@ spec: - "-c" - | if ! kubectl get awsnodetemplate workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - envsubst < /home/manifests/nodetemplate.yaml | kubectl apply -f - + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" | kubectl apply -f - fi if ! kubectl get provisioner workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - envsubst < /home/manifests/provisioner.yaml | kubectl apply -f - + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" | kubectl apply -f - fi env: - name: WORKFLOW_NAME diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml index c5b5019186..4f82e9d43e 100644 --- a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -41,7 +41,7 @@ spec: AWSNODETEMPLATE_TEMPLATE="nodetemplate.yaml" fi - ENVIRONMENT=$(kubectl get configmap global -o jsonpath="{.data.environment}") + ENVIRONMENT=$(kubectl -n default get configmap global -o jsonpath="{.data.environment}") RAW_WORKFLOWS=$(kubectl get workflows -n argo -o yaml) @@ -60,12 +60,13 @@ spec: if ! kubectl get awsnodetemplate workflow-$workflow_name >/dev/null 2>&1; then echo "No awsnodetemplate found for ${workflow_name}, creating one" - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USER_NAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f - + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f - fi if ! kubectl get provisioner workflow-$workflow_name >/dev/null 2>&1; then echo "No provisioner found for ${workflow_name}, creating one" - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USER_NAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f - + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f - + fi done restartPolicy: OnFailure From 6a10e7aa38a309733bac1989e894f452726f867d Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Fri, 8 Dec 2023 21:41:10 -0600 Subject: [PATCH 022/114] skip gender tags (#2376) * skip gender tags * dummy --- files/scripts/healdata/heal-cedar-data-ingest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index e95ab8604f..4a7d88c3cf 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -85,6 +85,8 @@ def update_filter_metadata(metadata_to_update): ] # Add any new tags from advSearchFilters for f in metadata_to_update["advSearchFilters"]: + if f["key"] == "Gender": + continue tag = {"name": f["value"], "category": f["key"]} if tag not in tags: tags.append(tag) From eaf04bf5b472b94558947f4842980576ce2ea1a5 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Mon, 11 Dec 2023 09:23:36 -0600 Subject: [PATCH 023/114] GPE-1081 (#2371) Co-authored-by: Edward Malinowski --- doc/s3-to-google-replication.md | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 doc/s3-to-google-replication.md diff --git a/doc/s3-to-google-replication.md b/doc/s3-to-google-replication.md new file mode 100644 index 0000000000..82d0374c7c --- /dev/null +++ b/doc/s3-to-google-replication.md @@ -0,0 +1,68 @@ +# S3 to Google Cloud Storage Replication Pipeline + +This document will guide you through setting up a replication pipeline from AWS S3 to Google Cloud Storage (GCS) using VPC Service Controls and Storage Transfer Service. This solution is compliant with security best practices, ensuring that data transfer between AWS S3 and GCS is secure and efficient. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Step-by-step Guide](#step-by-step-guide) + - [Setup VPC Service Controls](#setup-vpc-service-controls) + - [Initiate Storage Transfer Service](#initiate-storage-transfer-service) +- [Compliance Benefits](#compliance-benefits) +- [Cost Benefit Analysis](#cost-benefit-analysis) + +## Prerequisites + +1. **AWS account** with access to the S3 bucket. +2. **Google Cloud account** with permissions to create buckets in GCS and set up VPC Service Controls and Storage Transfer Service. +3. Familiarity with AWS IAM for S3 bucket access and Google Cloud IAM for GCS access. + +## Step-by-step Guide + +### Setup VPC Service Controls + +1. **Access the VPC Service Controls** in the Google Cloud Console. +2. **Create a new VPC Service Control perimeter**. + - Name the perimeter and choose the desired region. + - Add the necessary GCP services. Ensure to include `storagetransfer.googleapis.com` for Storage Transfer Service. +3. **Setup VPC Service Control Policy** to allow connections from AWS. + - Use the [documentation](https://cloud.google.com/vpc-service-controls/docs/set-up) to help set up. + +### Initiate Storage Transfer Service + +1. Navigate to **Storage Transfer Service** in the Google Cloud Console. +2. Click **Create Transfer Job**. +3. **Select Source**: Choose Amazon S3 bucket and provide the necessary details. + - Ensure to have necessary permissions for the S3 bucket in AWS IAM. +4. **Select Destination**: Choose your GCS bucket. +5. **Schedule & Advanced Settings**: Set the frequency and conditions for the transfer. Consider setting up notifications for job completion or errors. +6. **Review & Create**: Confirm the details and initiate the transfer job. + +## Compliance Benefits + +Setting up a secure replication pipeline from AWS S3 to GCS using VPC Service Controls and Storage Transfer Service offers the following compliance benefits: + +1. **Data Security**: The VPC Service Controls provide an additional layer of security by ensuring that the transferred data remains within a defined security perimeter, reducing potential data leak risks. +2. **Auditability**: Both AWS and GCS offer logging and monitoring tools that can provide audit trails for data transfer. This can help in meeting regulatory compliance requirements. +3. **Consistent Data Replication**: The Storage Transfer Service ensures that data in GCS is up to date with the source S3 bucket, which is essential for consistent backup and disaster recovery strategies. + +## Cost Benefit Analysis + +**Benefits**: + +1. **Data Redundancy**: Having data stored in multiple cloud providers can be a part of a robust disaster recovery strategy. +2. **Flexibility**: Replicating data to GCS provides flexibility in multi-cloud strategies, enabling seamless migrations or usage of GCP tools and services. +3. **Security**: Utilizing VPC Service Controls strengthens the security posture. + +**Costs**: + +1. **Data Transfer Costs**: Both AWS and Google Cloud might charge for data transfer. It's crucial to analyze the cost, especially for large data transfers. +2. **Storage Costs**: Storing data redundantly incurs additional storage costs in GCS. + +**Analysis**: + +To stay in compliance, we require multiple copies of our data in separate datacenters or clouds. After our security audit, we found the important of not keeping data in a single cloud. It may be expensive to transfer data from AWS to GCP and to store it in 2 clouds simultaniously, but if we need to, then this solution could be an easy way to achieve compliance. + +--- + +Please note that while this guide is based on the provided Google Cloud documentation, it's crucial to refer to the original [documentation](https://cloud.google.com/architecture/transferring-data-from-amazon-s3-to-cloud-storage-using-vpc-service-controls-and-storage-transfer-service) for the most accurate and up-to-date information. From df000c3fc49bbd916583002ce9cfdef9b1d27d45 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Mon, 11 Dec 2023 10:27:40 -0600 Subject: [PATCH 024/114] add csrf guppy (#2425) --- kube/services/revproxy/gen3.nginx.conf/guppy-service.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf b/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf index db2de58861..0e6b4f7e42 100644 --- a/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf @@ -1,4 +1,7 @@ location /guppy/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; From a6340ec429c38cf327ceafdd74a6ffcc9b1d93e2 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Wed, 13 Dec 2023 08:43:40 -0700 Subject: [PATCH 025/114] reverting guppy nginx config change (#2429) --- kube/services/revproxy/gen3.nginx.conf/guppy-service.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf b/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf index 0e6b4f7e42..db2de58861 100644 --- a/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf @@ -1,7 +1,4 @@ location /guppy/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; From f06e5ac03293088c3d3452e2bc0e4d7c8b7b0a14 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:42:46 -0700 Subject: [PATCH 026/114] adding node-to-node encryption (#2431) --- gen3/bin/create-es7-cluster.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gen3/bin/create-es7-cluster.sh b/gen3/bin/create-es7-cluster.sh index d18c4203f8..553dc26524 100644 --- a/gen3/bin/create-es7-cluster.sh +++ b/gen3/bin/create-es7-cluster.sh @@ -40,6 +40,7 @@ else --vpc-options "SubnetIds=${subnet_ids[*]},SecurityGroupIds=${security_groups[*]}" \ --access-policies "$access_policies" \ --encryption-at-rest-options "Enabled=true,KmsKeyId=$kms_key_id"\ + --node-to-node-encryption-options "Enabled=true" > /dev/null 2>&1 # Wait for the new cluster to be available @@ -60,4 +61,4 @@ else if [ $retry_count -eq $max_retries ]; then echo "New cluster creation may still be in progress. Please check the AWS Management Console for the status." fi -fi \ No newline at end of file +fi From 21cdceb78b978bc7005412508665e35ee6d5ec42 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Wed, 20 Dec 2023 13:31:43 -0600 Subject: [PATCH 027/114] Update ohdsi-webapi-deploy.yaml (#2432) --- kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml b/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml index 65d6ed38c9..a729ae7c41 100644 --- a/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml +++ b/kube/services/ohdsi-webapi/ohdsi-webapi-deploy.yaml @@ -83,7 +83,7 @@ spec: limits: memory: 4Gi - name: ohdsi-webapi-reverse-proxy - image: nginx:1.23 + image: 707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/nginx:1.23 ports: - containerPort: 80 volumeMounts: @@ -97,4 +97,4 @@ spec: cpu: 100m memory: 100Mi limits: - memory: 500Mi \ No newline at end of file + memory: 500Mi From fcc765f5ed9becdbb9172c7c0aad7d74fdc22d0b Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Fri, 22 Dec 2023 07:56:09 -0800 Subject: [PATCH 028/114] Upgrade jenkins (#2433) To 2.437 --- Docker/jenkins/Jenkins/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/jenkins/Jenkins/Dockerfile b/Docker/jenkins/Jenkins/Dockerfile index 8b8c58406b..94fcd3f12e 100644 --- a/Docker/jenkins/Jenkins/Dockerfile +++ b/Docker/jenkins/Jenkins/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.434-jdk21 +FROM jenkins/jenkins:2.437-jdk21 USER root From fcfebe0a2da425f406b6a78afefcfe0ae1cc9691 Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Wed, 3 Jan 2024 06:35:58 -0800 Subject: [PATCH 029/114] Upgrade jenkins (#2435) --- Docker/jenkins/Jenkins/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/jenkins/Jenkins/Dockerfile b/Docker/jenkins/Jenkins/Dockerfile index 94fcd3f12e..7cce68b58b 100644 --- a/Docker/jenkins/Jenkins/Dockerfile +++ b/Docker/jenkins/Jenkins/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.437-jdk21 +FROM jenkins/jenkins:2.439-jdk21 USER root From b859d3fc88c7b9c5e1a3dfda4c764a9cd16570b5 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Wed, 10 Jan 2024 12:00:25 -0600 Subject: [PATCH 030/114] PPS-588 add guppy csrf (#2430) * add guppy csrf * update msg --------- Co-authored-by: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> --- kube/services/revproxy/gen3.nginx.conf/guppy-service.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf b/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf index db2de58861..e6d66ec12e 100644 --- a/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf +++ b/kube/services/revproxy/gen3.nginx.conf/guppy-service.conf @@ -1,4 +1,8 @@ location /guppy/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check, make sure data-portal version >= 2023.12 or >= 5.19.0"; + } + proxy_connect_timeout 600s; proxy_send_timeout 600s; proxy_read_timeout 600s; From b7168ad9d2179a3d988d38bdccde3bc953ad714e Mon Sep 17 00:00:00 2001 From: Ajo Augustine Date: Tue, 16 Jan 2024 12:15:44 -0600 Subject: [PATCH 031/114] update service guppy memory limit (#2439) --- kube/services/guppy/guppy-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kube/services/guppy/guppy-deploy.yaml b/kube/services/guppy/guppy-deploy.yaml index 01a8905de7..c3e8d121c4 100644 --- a/kube/services/guppy/guppy-deploy.yaml +++ b/kube/services/guppy/guppy-deploy.yaml @@ -155,6 +155,6 @@ spec: resources: requests: cpu: 100m - memory: 128Mi + memory: 256Mi limits: - memory: 1200Mi + memory: 2000Mi From 1d594ce2d55842c2f295912858a6f3b5f8db09e8 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:42:55 -0700 Subject: [PATCH 032/114] Update web_whitelist (add elastic.co) (#2442) --- files/squid_whitelist/web_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index 0cbc0a6bc8..83070d3352 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -77,6 +77,7 @@ golang.org gopkg.in grafana.com grafana.github.io +helm.elastic.co http.us.debian.org ifconfig.io ingress.coralogix.us From 87704ae2a846d03903f58cfdc869dbd76e5d8aae Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:39:51 -0700 Subject: [PATCH 033/114] VPC CNI Migration From Calico (#2440) * Adding "internet: yes" label This will allow the "netpolicy-external-egress" networkpolicy to apply to revproxy and allow for all egress traffic. * updating the aws vpc cni version in our "kube-setup-system-services" script * adding "sudo mount -t bpf bpffs /sys/fs/bpf" to Karpenter node templates as AWS VPC CNI addon requires it in order to manage networkpolicies --- gen3/bin/kube-setup-system-services.sh | 2 +- kube/services/karpenter/nodeTemplateDefault.yaml | 1 + kube/services/karpenter/nodeTemplateGPU.yaml | 1 + kube/services/karpenter/nodeTemplateJupyter.yaml | 1 + kube/services/karpenter/nodeTemplateWorkflow.yaml | 1 + kube/services/revproxy/revproxy-deploy.yaml | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/gen3/bin/kube-setup-system-services.sh b/gen3/bin/kube-setup-system-services.sh index 609ee01c76..0afa7d5860 100644 --- a/gen3/bin/kube-setup-system-services.sh +++ b/gen3/bin/kube-setup-system-services.sh @@ -19,7 +19,7 @@ gen3_load "gen3/gen3setup" kubeproxy=${kubeproxy:-1.24.7} coredns=${coredns:-1.8.7} kubednsautoscaler=${kubednsautoscaler:-1.8.6} -cni=${cni:-1.12.2} +cni=${cni:-1.14.1} calico=${calico:-1.7.8} diff --git a/kube/services/karpenter/nodeTemplateDefault.yaml b/kube/services/karpenter/nodeTemplateDefault.yaml index a3dbf64802..107c5e6cca 100644 --- a/kube/services/karpenter/nodeTemplateDefault.yaml +++ b/kube/services/karpenter/nodeTemplateDefault.yaml @@ -37,6 +37,7 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + sudo mount -t bpf bpffs /sys/fs/bpf --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/karpenter/nodeTemplateGPU.yaml b/kube/services/karpenter/nodeTemplateGPU.yaml index 5270b697f3..c4fd535d74 100644 --- a/kube/services/karpenter/nodeTemplateGPU.yaml +++ b/kube/services/karpenter/nodeTemplateGPU.yaml @@ -37,6 +37,7 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + sudo mount -t bpf bpffs /sys/fs/bpf --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/karpenter/nodeTemplateJupyter.yaml b/kube/services/karpenter/nodeTemplateJupyter.yaml index 74f24926ac..bca4436d1f 100644 --- a/kube/services/karpenter/nodeTemplateJupyter.yaml +++ b/kube/services/karpenter/nodeTemplateJupyter.yaml @@ -37,6 +37,7 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + sudo mount -t bpf bpffs /sys/fs/bpf --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/karpenter/nodeTemplateWorkflow.yaml b/kube/services/karpenter/nodeTemplateWorkflow.yaml index ec2b81a60c..22c95aba11 100644 --- a/kube/services/karpenter/nodeTemplateWorkflow.yaml +++ b/kube/services/karpenter/nodeTemplateWorkflow.yaml @@ -37,6 +37,7 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" + sudo mount -t bpf bpffs /sys/fs/bpf --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/revproxy/revproxy-deploy.yaml b/kube/services/revproxy/revproxy-deploy.yaml index 9d5caab1b2..9f10ce90bd 100644 --- a/kube/services/revproxy/revproxy-deploy.yaml +++ b/kube/services/revproxy/revproxy-deploy.yaml @@ -21,6 +21,7 @@ spec: app: revproxy # allow access from workspaces userhelper: "yes" + internet: "yes" GEN3_DATE_LABEL spec: affinity: From 013ce31b4bb8eb49bbff2674f889d0ac28b6cc82 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Fri, 26 Jan 2024 12:50:15 -0500 Subject: [PATCH 034/114] This was still set to 5 for some reason (#2445) --- kube/services/argo/values.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml index 67fa05a09d..473f7041ea 100644 --- a/kube/services/argo/values.yaml +++ b/kube/services/argo/values.yaml @@ -1,5 +1,5 @@ controller: - parallelism: 5 + parallelism: 3 metricsConfig: # -- Enables prometheus metrics server enabled: true @@ -28,11 +28,11 @@ controller: } ] } - } + } resourceRateLimit: limit: 40 - burst: 4 + burst: 4 # -- enable persistence using postgres persistence: @@ -49,7 +49,7 @@ controller: port: 5432 database: GEN3_ARGO_DB_NAME tableName: argo_workflows - # # the database secrets must be in the same namespace of the controller + # # the database secrets must be in the same namespace of the controller userNameSecret: name: argo-db-creds key: db_username @@ -58,7 +58,7 @@ controller: key: db_password nodeStatusOffLoad: true - workflowDefaults: + workflowDefaults: spec: archiveLogs: true @@ -77,11 +77,11 @@ server: baseHref: "/argo/" # -- Extra arguments to provide to the Argo server binary, such as for disabling authentication. extraArgs: - - --auth-mode=server - - --auth-mode=client + - --auth-mode=server + - --auth-mode=client extraEnv: - - name: ARGO_HTTP1 - value: "true" + - name: ARGO_HTTP1 + value: "true" resources: requests: memory: 8Gi From 368eeb0f3046ae90f4c7ee01658fe30752daf9c9 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Wed, 31 Jan 2024 10:31:27 -0600 Subject: [PATCH 035/114] chore(karpenter-upgrade): Updraded karpenter to a supported version when running eks 1.25+ (#2450) Co-authored-by: Edward Malinowski --- gen3/bin/kube-setup-karpenter.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh index 8ba8ed9d97..4dba4eb40b 100644 --- a/gen3/bin/kube-setup-karpenter.sh +++ b/gen3/bin/kube-setup-karpenter.sh @@ -24,7 +24,9 @@ gen3_deploy_karpenter() { karpenter=$(g3k_config_lookup .global.karpenter_version) fi export clusterversion=`kubectl version --short -o json | jq -r .serverVersion.minor` - if [ "${clusterversion}" = "24+" ]; then + if [ "${clusterversion}" = "25+" ]; then + karpenter=${karpenter:-v0.27.0} + elif [ "${clusterversion}" = "24+" ]; then karpenter=${karpenter:-v0.24.0} else karpenter=${karpenter:-v0.22.0} From 12baa786e1771e0d4f5533844fee978fd80a51a8 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:32:54 -0700 Subject: [PATCH 036/114] adding "migrate to vpc cni" script to cloud-auto (#2446) * adding "migrate to vpc cni" script to cloud-auto * updating the migrate to vpc script * changing to master branch * removing uneeded comments * Updated karpenter templates * adding changes --------- Co-authored-by: Edward Malinowski Co-authored-by: emalinowski --- gen3/bin/migrate-to-vpc-cni.sh | 138 ++++++++++++++++++ .../karpenter/nodeTemplateDefault.yaml | 8 +- kube/services/karpenter/nodeTemplateGPU.yaml | 7 +- .../karpenter/nodeTemplateJupyter.yaml | 7 +- .../karpenter/nodeTemplateWorkflow.yaml | 7 +- 5 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 gen3/bin/migrate-to-vpc-cni.sh diff --git a/gen3/bin/migrate-to-vpc-cni.sh b/gen3/bin/migrate-to-vpc-cni.sh new file mode 100644 index 0000000000..510d9ebeff --- /dev/null +++ b/gen3/bin/migrate-to-vpc-cni.sh @@ -0,0 +1,138 @@ +#!/bin/bash + +source "${GEN3_HOME}/gen3/lib/utils.sh" +gen3_load "gen3/gen3setup" + +#Get the K8s NS +ctx="$(g3kubectl config current-context)" +ctxNamespace="$(g3kubectl config view -ojson | jq -r ".contexts | map(select(.name==\"$ctx\")) | .[0] | .context.namespace")" + +# Set the cluster name variable +CLUSTER_NAME=`gen3 api environment` + +# Check if in default ns +if [[ ("$ctxNamespace" != "default" && "$ctxNamespace" != "null") ]]; then + gen3_log_err "Namespace must be default" + exit 1 +fi + +# Cd into Cloud-automation repo and pull the latest from master +gen3_log_info "Pulling the latest from Cloud-Auto" +cd /home/$CLUSTER_NAME/cloud-automation || { gen3_log_err "Cloud-automation repo not found"; exit 1; } +#### Change to master +git checkout master || { gen3_log_err "Failed to checkout master branch"; exit 1; } +git pull || { gen3_log_err "Failed to pull from the repository"; exit 1; } + +# Update the Karpenter Node Template +gen3_log_info "Apply new Karpenter Node Template" +if [[ -d $(g3k_manifest_init)/$(g3k_hostname)/manifests/karpenter ]]; then + gen3_log_info "Karpenter setup in manifest. Open a cdismanifest PR and add this line to aws node templates: https://github.com/uc-cdis/cloud-automation/blob/master/kube/services/karpenter/nodeTemplateDefault.yaml#L40" + while true; do + read -p "Have you updated your manifest? (yes/no): " yn + case $yn in + [Yy]* ) + gen3_log_info "Proceeding with Karpenter deployment..." + gen3 kube-setup-karpenter deploy --force || { gen3_log_err "kube-setup-karpenter failed"; exit 1; } + break + ;; + [Nn]* ) + gen3_log_info "Please update the cdismanifest before proceeding." + exit 1 + ;; + * ) + gen3_log_info "Please answer yes or no." + ;; + esac + done +else + gen3 kube-setup-karpenter deploy --force || { gen3_log_err "kube-setup-karpenter failed"; exit 1; } +fi + +# Cordon all the nodes before running gen3 roll all" +gen3_log_info "Cordoning all nodes" +kubectl get nodes --no-headers -o custom-columns=":metadata.name" | grep -v '^fargate' | xargs -I{} kubectl cordon {} + +# Run a "gen3 roll all" so all nodes use the new mounted BPF File System +gen3_log_info "Cycling all the nodes by running gen3 roll all" +gen3 roll all --fast || exit 1 + +# Confirm that all nodes have been rotated +while true; do + read -p "Roll all complete. Have all cordoned nodes been rotated? (yes/no): " yn + case $yn in + [Yy]* ) + gen3_log_info "Continuing with script..." + break + ;; + [Nn]* ) + gen3_log_info "Please drain any remaining nodes with 'kubectl drain --ignore-daemonsets --delete-emptydir-data'" + ;; + * ) + gen3_log_info "Please answer yes or no." + ;; + esac +done + + +# Delete all existing network policies +gen3_log_info "Deleting networkpolicies" +kubectl delete networkpolicies --all + +# Delete all Calico related resources from the “kube-system” namespace +gen3_log_info "Deleting all Calico related resources" +kubectl get deployments -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete deployment -n kube-system +kubectl get daemonsets -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete daemonset -n kube-system +kubectl get services -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete service -n kube-system +kubectl get replicasets -n kube-system | grep calico | awk '{print $1}' | xargs kubectl delete replicaset -n kube-system + +# Backup the current VPC CNI configuration in case of rollback +gen3_log_info "Backing up current VPC CNI Configuration..." +kubectl get daemonset aws-node -n kube-system -o yaml > aws-k8s-cni-old.yaml || { gen3_log_err "Error backig up VPC CNI configuration"; exit 1; } + +# Check to ensure we are not using an AWS plugin to manage the VPC CNI Plugin +if aws eks describe-addon --cluster-name "$CLUSTER_NAME" --addon-name vpc-cni --query addon.addonVersion --output text 2>/dev/null; then + gen3_log_err "Error: VPC CNI Plugin is managed by AWS. Please log into the AWS UI and delete the VPC CNI Plugin in Amazon EKS, then re-run this script." + exit 1 +else + gen3_log_info "No managed VPC CNI Plugin found, proceeding with the script." +fi + +# Apply the new VPC CNI Version +gen3_log_info "Applying new version of VPC CNI" +g3kubectl apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v1.14.1/config/master/aws-k8s-cni.yaml || { gen3_log_err "Failed to apply new VPC CNI version"; exit 1; } + +# Check the version to make sure it updated +NEW_VERSION=$(kubectl describe daemonset aws-node --namespace kube-system | grep amazon-k8s-cni: | cut -d : -f 3) +gen3_log_info "Current version of aws-k8s-cni is: $NEW_VERSION" +if [ "$NEW_VERSION" != "v1.14.1" ]; then + gen3_log_info "The version of aws-k8s-cni has not been updated correctly." + exit 1 +fi + +# Edit the amazon-vpc-cni configmap to enable network policy controller +gen3_log_info "Enabling NetworkPolicies in VPC CNI Configmap" +kubectl patch configmap -n kube-system amazon-vpc-cni --type merge -p '{"data":{"enable-network-policy-controller":"true"}}' || { gen3_log_err "Configmap patch failed"; exit 1; } + +# Edit the aws-node daemonset +gen3_log_info "Enabling NetworkPolicies in aws-node Daemonset" +kubectl patch daemonset aws-node -n kube-system --type=json -p='[{"op": "add", "path": "/spec/template/spec/containers/1/args", "value": ["--enable-network-policy=true", "--enable-ipv6=false", "--enable-cloudwatch-logs=false", "--metrics-bind-addr=:8162", "--health-probe-bind-addr=:8163"]}]' || { gen3_log_err "Daemonset edit failed"; exit 1; } + +# Ensure all the aws-nodes are running +kubectl get pods -n kube-system | grep aws +while true; do + read -p "Do all the aws-node pods in the kube-system ns have 2/2 containers running? (yes/no): " yn + case $yn in + [Yy]* ) + gen3_log_info "Running kube-setup-networkpolicy..." + gen3 kube-setup-networkpolicy || exit 1 + break + ;; + [Nn]* ) + gen3_log_err "Look at aws-node logs to figure out what went wrong. View this document for more details: https://docs.google.com/document/d/1fcBTciQSSwjvHktEnO_7EObY-xR_EvJ2NtgUa70wvL8" + gen3_log_info "Rollback instructions are also available in the above document" + ;; + * ) + gen3_log_info "Please answer yes or no." + ;; + esac +done \ No newline at end of file diff --git a/kube/services/karpenter/nodeTemplateDefault.yaml b/kube/services/karpenter/nodeTemplateDefault.yaml index 107c5e6cca..114de5abad 100644 --- a/kube/services/karpenter/nodeTemplateDefault.yaml +++ b/kube/services/karpenter/nodeTemplateDefault.yaml @@ -37,11 +37,17 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - sudo mount -t bpf bpffs /sys/fs/bpf --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" + mounts: + - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] + + --BOUNDARY + + Content-Type: text/cloud-config; charset="us-ascii" + power_state: delay: now mode: reboot diff --git a/kube/services/karpenter/nodeTemplateGPU.yaml b/kube/services/karpenter/nodeTemplateGPU.yaml index c4fd535d74..cd3eb7386e 100644 --- a/kube/services/karpenter/nodeTemplateGPU.yaml +++ b/kube/services/karpenter/nodeTemplateGPU.yaml @@ -37,7 +37,12 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - sudo mount -t bpf bpffs /sys/fs/bpf + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + mounts: + - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/karpenter/nodeTemplateJupyter.yaml b/kube/services/karpenter/nodeTemplateJupyter.yaml index bca4436d1f..af2da84368 100644 --- a/kube/services/karpenter/nodeTemplateJupyter.yaml +++ b/kube/services/karpenter/nodeTemplateJupyter.yaml @@ -37,7 +37,12 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - sudo mount -t bpf bpffs /sys/fs/bpf + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + mounts: + - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/karpenter/nodeTemplateWorkflow.yaml b/kube/services/karpenter/nodeTemplateWorkflow.yaml index 22c95aba11..8609d0746a 100644 --- a/kube/services/karpenter/nodeTemplateWorkflow.yaml +++ b/kube/services/karpenter/nodeTemplateWorkflow.yaml @@ -37,7 +37,12 @@ spec: sudo dracut -f # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - sudo mount -t bpf bpffs /sys/fs/bpf + + --BOUNDARY + Content-Type: text/cloud-config; charset="us-ascii" + + mounts: + - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" From c6358c90bf58ca2b38eeb290ef721f0ed40ae2dc Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Thu, 1 Feb 2024 12:59:21 -0800 Subject: [PATCH 037/114] Update jenkins version (#2455) --- Docker/jenkins/Jenkins/Dockerfile | 2 +- Docker/jenkins/Jenkins2/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Docker/jenkins/Jenkins/Dockerfile b/Docker/jenkins/Jenkins/Dockerfile index 7cce68b58b..04ebe5864a 100644 --- a/Docker/jenkins/Jenkins/Dockerfile +++ b/Docker/jenkins/Jenkins/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.439-jdk21 +FROM jenkins/jenkins:2.426.3-lts-jdk21 USER root diff --git a/Docker/jenkins/Jenkins2/Dockerfile b/Docker/jenkins/Jenkins2/Dockerfile index 9976a07c20..e6b73bc76d 100644 --- a/Docker/jenkins/Jenkins2/Dockerfile +++ b/Docker/jenkins/Jenkins2/Dockerfile @@ -1,4 +1,4 @@ -FROM jenkins/jenkins:2.415-jdk11 +FROM jenkins/jenkins:2.426.3-lts-jdk21 USER root From 916d1ca7f7a47103eee518574c16a8aa2b2c1f7c Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Fri, 2 Feb 2024 12:35:42 -0600 Subject: [PATCH 038/114] update script (#2454) Co-authored-by: Michael Lukowski --- files/scripts/healdata/heal-cedar-data-ingest.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index 4a7d88c3cf..71575e3c56 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -24,11 +24,14 @@ "Questionnaire/Survey/Assessment - unvalidated instrument": "Questionnaire/Survey/Assessment", "Cis Male": "Male", "Cis Female": "Female", - "Trans Male": "Female-to-male transsexual", - "Trans Female": "Male-to-female transsexual", - "Agender, Non-binary, gender non-conforming": "Other", - "Gender Queer": "Other", - "Intersex": "Intersexed", + "Trans Male": "Transgender man/trans man/female-to-male (FTM)", + "Female-to-male transsexual": "Transgender man/trans man/female-to-male (FTM)", + "Trans Female": "Transgender woman/trans woman/male-to-female (MTF)", + "Male-to-female transsexual": "Transgender woman/trans woman/male-to-female (MTF)", + "Agender, Non-binary, gender non-conforming": "Genderqueer/gender nonconforming/neither exclusively male nor female", + "Gender Queer": "Genderqueer/gender nonconforming/neither exclusively male nor female", + "Intersex": "Genderqueer/gender nonconforming/neither exclusively male nor female", + "Intersexed": "Genderqueer/gender nonconforming/neither exclusively male nor female", "Buisness Development": "Business Development" } From f6ded7a98e76135657ef8b2e2c0e0455e571414d Mon Sep 17 00:00:00 2001 From: emalinowski Date: Mon, 5 Feb 2024 13:46:57 -0600 Subject: [PATCH 039/114] Chore/squid al23 (#2456) * chore(squid-al23): Updated squid bootstrap script to work with al23 * chore(squid-al23): Updated squid bootstrap script to work with al23 * chore(squid-al23): Updated squid bootstrap script to work with al23 * chore(squid-al23): Updated squid bootstrap script to work with al23 --------- Co-authored-by: Edward Malinowski --- flavors/squid_auto/squid_running_on_docker.sh | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh index 05607f3044..7504cc35ee 100644 --- a/flavors/squid_auto/squid_running_on_docker.sh +++ b/flavors/squid_auto/squid_running_on_docker.sh @@ -8,6 +8,9 @@ DISTRO=$(awk -F '[="]*' '/^NAME/ { print $2 }' < /etc/os-release) WORK_USER="ubuntu" if [[ $DISTRO == "Amazon Linux" ]]; then WORK_USER="ec2-user" + if [[ $(awk -F '[="]*' '/^VERSION_ID/ { print $2 }' < /etc/os-release) == "2023" ]]; then + DISTRO="al2023" + fi fi HOME_FOLDER="/home/${WORK_USER}" SUB_FOLDER="${HOME_FOLDER}/cloud-automation" @@ -201,8 +204,10 @@ function install_awslogs { if [[ $DISTRO == "Ubuntu" ]]; then wget ${AWSLOGS_DOWNLOAD_URL} -O amazon-cloudwatch-agent.deb dpkg -i -E ./amazon-cloudwatch-agent.deb - else + elif [[ $DISTRO == "Amazon Linux" ]]; then sudo yum install amazon-cloudwatch-agent nc -y + elif [[ $DISTRO == "al2023" ]]; then + sudo dnf install amazon-cloudwatch-agent nc -y fi # Configure the AWS logs @@ -292,6 +297,19 @@ function main(){ --volume ${SQUID_CACHE_DIR}:${SQUID_CACHE_DIR} \ --volume ${SQUID_CONFIG_DIR}:${SQUID_CONFIG_DIR}:ro \ quay.io/cdis/squid:${SQUID_IMAGE_TAG} + + max_attempts=3 + attempt_counter=0 + while [ $attempt_counter -lt $max_attempts ]; do + sleep 10 + if [[ -z "$(sudo lsof -i:3128)" ]]; then + echo "Squid not healthy, restarting." + docker restart squid + else + echo "Squid healthy" + break + fi + done } main From f24a0dacc4053058d5bb949d5e21c938e68b82eb Mon Sep 17 00:00:00 2001 From: emalinowski Date: Tue, 6 Feb 2024 10:45:03 -0600 Subject: [PATCH 040/114] chore(squid-al23): Updated squid bootstrap script to work with al23 (#2457) Co-authored-by: Edward Malinowski --- flavors/squid_auto/squid_running_on_docker.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh index 7504cc35ee..2b0f07e459 100644 --- a/flavors/squid_auto/squid_running_on_docker.sh +++ b/flavors/squid_auto/squid_running_on_docker.sh @@ -298,7 +298,7 @@ function main(){ --volume ${SQUID_CONFIG_DIR}:${SQUID_CONFIG_DIR}:ro \ quay.io/cdis/squid:${SQUID_IMAGE_TAG} - max_attempts=3 + max_attempts=10 attempt_counter=0 while [ $attempt_counter -lt $max_attempts ]; do sleep 10 @@ -307,7 +307,6 @@ function main(){ docker restart squid else echo "Squid healthy" - break fi done } From 57120c9e9fea0b79c411f3d88844fca8987fb11f Mon Sep 17 00:00:00 2001 From: emalinowski Date: Tue, 6 Feb 2024 12:26:06 -0600 Subject: [PATCH 041/114] chore(tigera-helm): Updated squid to support calico chart repo (#2458) Co-authored-by: Edward Malinowski --- files/squid_whitelist/web_wildcard_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist index 44f4680971..3dca3946ad 100644 --- a/files/squid_whitelist/web_wildcard_whitelist +++ b/files/squid_whitelist/web_wildcard_whitelist @@ -100,6 +100,7 @@ .sourceforge.net .southsideweekly.com .theanvil.io +.tigera.io .twistlock.com .ubuntu.com .ucsc.edu From 8c00a2306cba07ff3894aa4688ac14cbdfc6724e Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Tue, 6 Feb 2024 15:53:14 -0600 Subject: [PATCH 042/114] Update web_whitelist --- files/squid_whitelist/web_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index 83070d3352..625c20b299 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -34,6 +34,7 @@ cernvm.cern.ch charts.bitnami.com charts.helm.sh cloud.r-project.org +coredns.github.io coreos.com covidstoplight.org cpan.mirrors.tds.net From 36ea4d58f7f16cb5a05a844dd6db545036994f4d Mon Sep 17 00:00:00 2001 From: emalinowski Date: Wed, 7 Feb 2024 10:09:09 -0600 Subject: [PATCH 043/114] fix(alb-policy): policy fix (#2459) Co-authored-by: Edward Malinowski --- gen3/bin/kube-setup-ingress.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/gen3/bin/kube-setup-ingress.sh b/gen3/bin/kube-setup-ingress.sh index d0bcff9a4b..b75470f733 100644 --- a/gen3/bin/kube-setup-ingress.sh +++ b/gen3/bin/kube-setup-ingress.sh @@ -232,6 +232,28 @@ gen3_ingress_setup_role() { } } }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags" + ], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "StringEquals": { + "elasticloadbalancing:CreateAction": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, { "Effect": "Allow", "Action": [ @@ -329,4 +351,4 @@ g3kubectl apply -f "${GEN3_HOME}/kube/services/revproxy/revproxy-service.yaml" envsubst <$scriptDir/ingress.yaml | g3kubectl apply -f - if [ "$deployWaf" = true ]; then gen3_ingress_setup_waf -fi \ No newline at end of file +fi From 301744b69a069cfddccd8a67fc77aa5362955dd0 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Wed, 7 Feb 2024 10:51:02 -0700 Subject: [PATCH 044/114] removing "--short" flag as it is now deprecated (#2462) --- gen3/bin/kube-setup-karpenter.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh index 4dba4eb40b..c8762c2e58 100644 --- a/gen3/bin/kube-setup-karpenter.sh +++ b/gen3/bin/kube-setup-karpenter.sh @@ -23,7 +23,7 @@ gen3_deploy_karpenter() { if g3k_config_lookup .global.karpenter_version; then karpenter=$(g3k_config_lookup .global.karpenter_version) fi - export clusterversion=`kubectl version --short -o json | jq -r .serverVersion.minor` + export clusterversion=`kubectl version -o json | jq -r .serverVersion.minor` if [ "${clusterversion}" = "25+" ]; then karpenter=${karpenter:-v0.27.0} elif [ "${clusterversion}" = "24+" ]; then From d01c0fa1fd8b612ab7b03c1d9a408e5e8bf36656 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Wed, 7 Feb 2024 10:53:02 -0700 Subject: [PATCH 045/114] reverting the BPF (#2461) --- kube/services/karpenter/nodeTemplateDefault.yaml | 8 ++++---- kube/services/karpenter/nodeTemplateGPU.yaml | 8 ++++---- kube/services/karpenter/nodeTemplateJupyter.yaml | 8 ++++---- kube/services/karpenter/nodeTemplateWorkflow.yaml | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/kube/services/karpenter/nodeTemplateDefault.yaml b/kube/services/karpenter/nodeTemplateDefault.yaml index 114de5abad..6ba8b3a0f7 100644 --- a/kube/services/karpenter/nodeTemplateDefault.yaml +++ b/kube/services/karpenter/nodeTemplateDefault.yaml @@ -38,11 +38,11 @@ spec: # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - --BOUNDARY - Content-Type: text/cloud-config; charset="us-ascii" + # --BOUNDARY + # Content-Type: text/cloud-config; charset="us-ascii" - mounts: - - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] + # mounts: + # - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] --BOUNDARY diff --git a/kube/services/karpenter/nodeTemplateGPU.yaml b/kube/services/karpenter/nodeTemplateGPU.yaml index cd3eb7386e..925e7a9a08 100644 --- a/kube/services/karpenter/nodeTemplateGPU.yaml +++ b/kube/services/karpenter/nodeTemplateGPU.yaml @@ -38,11 +38,11 @@ spec: # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - --BOUNDARY - Content-Type: text/cloud-config; charset="us-ascii" + # --BOUNDARY + # Content-Type: text/cloud-config; charset="us-ascii" - mounts: - - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] + # mounts: + # - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/karpenter/nodeTemplateJupyter.yaml b/kube/services/karpenter/nodeTemplateJupyter.yaml index af2da84368..1c8970ad64 100644 --- a/kube/services/karpenter/nodeTemplateJupyter.yaml +++ b/kube/services/karpenter/nodeTemplateJupyter.yaml @@ -38,11 +38,11 @@ spec: # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - --BOUNDARY - Content-Type: text/cloud-config; charset="us-ascii" + # --BOUNDARY + # Content-Type: text/cloud-config; charset="us-ascii" - mounts: - - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] + # mounts: + # - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" diff --git a/kube/services/karpenter/nodeTemplateWorkflow.yaml b/kube/services/karpenter/nodeTemplateWorkflow.yaml index 8609d0746a..6e47b22f97 100644 --- a/kube/services/karpenter/nodeTemplateWorkflow.yaml +++ b/kube/services/karpenter/nodeTemplateWorkflow.yaml @@ -38,11 +38,11 @@ spec: # configure grub sudo /sbin/grubby --update-kernel=ALL --args="fips=1" - --BOUNDARY - Content-Type: text/cloud-config; charset="us-ascii" + # --BOUNDARY + # Content-Type: text/cloud-config; charset="us-ascii" - mounts: - - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] + # mounts: + # - ['fstype': 'bpf', 'mountpoint': '/sys/fs/bpf', 'opts': 'rw,relatime'] --BOUNDARY Content-Type: text/cloud-config; charset="us-ascii" From 21000363e7c3d562de45dfa7d07ffd24260a300d Mon Sep 17 00:00:00 2001 From: emalinowski Date: Wed, 7 Feb 2024 12:54:00 -0600 Subject: [PATCH 046/114] chore(remove-coredns-autoscaler): Removed dns autoscaler deployment (#2460) Co-authored-by: Edward Malinowski --- gen3/bin/kube-roll-all.sh | 2 +- gen3/bin/kube-setup-system-services.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gen3/bin/kube-roll-all.sh b/gen3/bin/kube-roll-all.sh index c9cec5a25d..6a67f2bdd2 100644 --- a/gen3/bin/kube-roll-all.sh +++ b/gen3/bin/kube-roll-all.sh @@ -274,7 +274,7 @@ if [[ "$GEN3_ROLL_FAST" != "true" ]]; then else gen3 kube-setup-autoscaler & fi - gen3 kube-setup-kube-dns-autoscaler & + #gen3 kube-setup-kube-dns-autoscaler & gen3 kube-setup-metrics deploy || true gen3 kube-setup-tiller || true # diff --git a/gen3/bin/kube-setup-system-services.sh b/gen3/bin/kube-setup-system-services.sh index 0afa7d5860..c26a04cb5d 100644 --- a/gen3/bin/kube-setup-system-services.sh +++ b/gen3/bin/kube-setup-system-services.sh @@ -39,7 +39,7 @@ calico_yaml="https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${calico} g3kubectl set image daemonset.apps/kube-proxy -n kube-system kube-proxy=${kube_proxy_image} g3kubectl set image --namespace kube-system deployment.apps/coredns coredns=${coredns_image} -g3k_kv_filter "${GEN3_HOME}/kube/services/kube-dns-autoscaler/dns-horizontal-autoscaler.yaml" SERVICE "coredns" IMAGE "$kubednsautoscaler_image" | g3kubectl apply -f - +#g3k_kv_filter "${GEN3_HOME}/kube/services/kube-dns-autoscaler/dns-horizontal-autoscaler.yaml" SERVICE "coredns" IMAGE "$kubednsautoscaler_image" | g3kubectl apply -f - g3kubectl apply -f ${cni_image} g3kubectl apply -f ${calico_yaml} From 5f7aed08fa60ce1364c8016ff95ad9df6d34bfa1 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Wed, 7 Feb 2024 12:55:03 -0600 Subject: [PATCH 047/114] fix(squid-cron): Added crontab to al23 squid (#2463) Co-authored-by: Edward Malinowski --- flavors/squid_auto/squid_running_on_docker.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh index 2b0f07e459..557809b695 100644 --- a/flavors/squid_auto/squid_running_on_docker.sh +++ b/flavors/squid_auto/squid_running_on_docker.sh @@ -63,6 +63,8 @@ fi function install_basics(){ if [[ $DISTRO == "Ubuntu" ]]; then apt -y install atop + elif [[ $DISTRO == "al2023" ]]; then + sudo dnf install cronie nc -y fi } @@ -207,7 +209,7 @@ function install_awslogs { elif [[ $DISTRO == "Amazon Linux" ]]; then sudo yum install amazon-cloudwatch-agent nc -y elif [[ $DISTRO == "al2023" ]]; then - sudo dnf install amazon-cloudwatch-agent nc -y + sudo dnf install amazon-cloudwatch-agent -y fi # Configure the AWS logs From 0ebd73040e0c3eba67de10b85061a60c969890c9 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 9 Feb 2024 09:39:38 -0700 Subject: [PATCH 048/114] Update ingress.yaml to use newer fips based tls policy (#2447) --- kube/services/ingress/ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/ingress/ingress.yaml b/kube/services/ingress/ingress.yaml index 65916679a7..3f1f312592 100644 --- a/kube/services/ingress/ingress.yaml +++ b/kube/services/ingress/ingress.yaml @@ -11,7 +11,7 @@ metadata: alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=600 alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-2021-06 + alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04 spec: ingressClassName: alb rules: From 1bf159a57d59dac52544c65fc01f057f53e0b0a7 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Fri, 9 Feb 2024 14:38:55 -0600 Subject: [PATCH 049/114] fix syntax (#2464) Co-authored-by: Edward Malinowski --- flavors/squid_auto/squid_running_on_docker.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh index 557809b695..94fe22122b 100644 --- a/flavors/squid_auto/squid_running_on_docker.sh +++ b/flavors/squid_auto/squid_running_on_docker.sh @@ -303,6 +303,7 @@ function main(){ max_attempts=10 attempt_counter=0 while [ $attempt_counter -lt $max_attempts ]; do + ((attempt_counter++)) sleep 10 if [[ -z "$(sudo lsof -i:3128)" ]]; then echo "Squid not healthy, restarting." From 0f98195cf8b794b044c063b19281a39aca43b3ee Mon Sep 17 00:00:00 2001 From: emalinowski Date: Fri, 9 Feb 2024 15:32:21 -0600 Subject: [PATCH 050/114] prevent reboot (#2465) Co-authored-by: Edward Malinowski --- flavors/squid_auto/squid_running_on_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh index 94fe22122b..2d7cf8e688 100644 --- a/flavors/squid_auto/squid_running_on_docker.sh +++ b/flavors/squid_auto/squid_running_on_docker.sh @@ -303,7 +303,7 @@ function main(){ max_attempts=10 attempt_counter=0 while [ $attempt_counter -lt $max_attempts ]; do - ((attempt_counter++)) + #((attempt_counter++)) sleep 10 if [[ -z "$(sudo lsof -i:3128)" ]]; then echo "Squid not healthy, restarting." From 8e02b097b822927ae1c2768fa93b828e2150d732 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Mon, 12 Feb 2024 16:07:45 -0500 Subject: [PATCH 051/114] Feat/argo va testing revproxy modifications (#2466) * It can't be that easy * It wasn't that easy. What we're going to do is use the manifest to tell us what namespace to look in, and then look there * Dumb typo * Yet another dumb typo * Moving to a different setup, where specify the argo server URL in the manifest. This is more flexible * I'm blaming the last commit on Monday brain * It's a .conf file, not a yaml file * Need to update in both places --- gen3/bin/kube-setup-revproxy.sh | 17 ++++++++--------- ...o-workflows-server.conf => argo-server.conf} | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) rename kube/services/revproxy/gen3.nginx.conf/{argo-argo-workflows-server.conf => argo-server.conf} (86%) diff --git a/gen3/bin/kube-setup-revproxy.sh b/gen3/bin/kube-setup-revproxy.sh index fcc2ef3b73..5db9850a18 100644 --- a/gen3/bin/kube-setup-revproxy.sh +++ b/gen3/bin/kube-setup-revproxy.sh @@ -111,15 +111,14 @@ for name in $(g3kubectl get services -o json | jq -r '.items[] | .metadata.name' fi done -if g3kubectl get namespace argo > /dev/null 2>&1; -then - for argo in $(g3kubectl get services -n argo -o jsonpath='{.items[*].metadata.name}'); - do - filePath="$scriptDir/gen3.nginx.conf/${argo}.conf" - if [[ -f "$filePath" ]]; then - confFileList+=("--from-file" "$filePath") - fi - done + +if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then + argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url) + g3k_kv_filter "${scriptDir}/gen3.nginx.conf/argo-server.conf" SERVICE_URL "${argo_server_service_url}" > /tmp/argo-server-with-url.conf + filePath="/tmp/argo-server-with-url.conf" + if [[ -f "$filePath" ]]; then + confFileList+=("--from-file" "$filePath") + fi fi if g3kubectl get namespace argocd > /dev/null 2>&1; diff --git a/kube/services/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf b/kube/services/revproxy/gen3.nginx.conf/argo-server.conf similarity index 86% rename from kube/services/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf rename to kube/services/revproxy/gen3.nginx.conf/argo-server.conf index cb8def3aa5..1cdd4608c6 100644 --- a/kube/services/revproxy/gen3.nginx.conf/argo-argo-workflows-server.conf +++ b/kube/services/revproxy/gen3.nginx.conf/argo-server.conf @@ -7,7 +7,7 @@ auth_request /gen3-authz; set $proxy_service "argo"; - set $upstream http://argo-argo-workflows-server.argo.svc.cluster.local:2746; + set $upstream SERVICE_URL; rewrite ^/argo/(.*) /$1 break; From e081b2385cffdfb832063fbd7c3930f0310ed738 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Mon, 12 Feb 2024 17:02:16 -0600 Subject: [PATCH 052/114] fix(karpenter-policy): Added kms permissions for karpenter (#2467) * fix(karpenter-policy): Added kms permissions for karpenter * fix(karpenter-policy): Added kms permissions for karpenter * fix(karpenter-policy): Added kms permissions for karpenter --------- Co-authored-by: Edward Malinowski Co-authored-by: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> --- gen3/bin/kube-setup-karpenter.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh index c8762c2e58..2737ed6eeb 100644 --- a/gen3/bin/kube-setup-karpenter.sh +++ b/gen3/bin/kube-setup-karpenter.sh @@ -79,6 +79,14 @@ gen3_deploy_karpenter() { "Effect": "Allow", "Resource": "*", "Sid": "ConditionalEC2Termination" + }, + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "kms:*" + ], + "Resource": "*" } ], "Version": "2012-10-17" From 11b94fb4e86ac779bf855a451bd16c7e7ecbe5d1 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Tue, 13 Feb 2024 09:47:57 -0600 Subject: [PATCH 053/114] Feat/argo wrapper namespace config (#2468) * feat: argo-wrapper-namespace-config update * feat: some extra updates * fix: name for file * feat: updates --- gen3/bin/kube-setup-argo-wrapper.sh | 13 ++++++++++++- kube/services/argo-wrapper/argo-wrapper-deploy.yaml | 9 ++++++++- kube/services/argo-wrapper/config.ini | 4 ++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 kube/services/argo-wrapper/config.ini diff --git a/gen3/bin/kube-setup-argo-wrapper.sh b/gen3/bin/kube-setup-argo-wrapper.sh index 5727a703e0..306050b124 100644 --- a/gen3/bin/kube-setup-argo-wrapper.sh +++ b/gen3/bin/kube-setup-argo-wrapper.sh @@ -19,5 +19,16 @@ if [[ -z "$GEN3_SOURCE_ONLY" ]]; then gen3 roll argo-wrapper g3kubectl apply -f "${GEN3_HOME}/kube/services/argo-wrapper/argo-wrapper-service.yaml" + if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then + argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url) + + export ARGO_HOST=${argo_server_service_url} + export ARGO_NAMESPACE=argo-$(gen3 db namespace) + envsubst <"${GEN3_HOME}/kube/services/argo-wrapper/config.ini" > /tmp/config.ini + + g3kubectl delete configmap argo-wrapper-namespace-config + g3kubectl create configmap argo-wrapper-namespace-config --from-file /tmp/config.ini + fi + gen3_log_info "the argo-wrapper service has been deployed onto the kubernetes cluster" -fi \ No newline at end of file +fi diff --git a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml index 65f68d98ab..00d118746c 100644 --- a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml +++ b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml @@ -58,7 +58,10 @@ spec: configMap: name: manifest-argo optional: true - + - name: argo-wrapper-namespace-config + configMap: + name: argo-wrapper-namespace-config + containers: - name: argo-wrapper GEN3_ARGO-WRAPPER_IMAGE @@ -70,3 +73,7 @@ spec: readOnly: true mountPath: /argo.json subPath: argo.json + - name: argo-wrapper-namespace-config + readOnly: true + mountPath: /src/config.ini + subPath: config.ini diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini new file mode 100644 index 0000000000..334438ac20 --- /dev/null +++ b/kube/services/argo-wrapper/config.ini @@ -0,0 +1,4 @@ +[DEFAULT] +ARGO_ACCESS_METHOD = access +ARGO_HOST = $ARGO_HOST +ARGO_NAMESPACE = $ARGO_NAMESPACE From 464f6ebc68f1481c83597920ae411226cdff7ee0 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Wed, 14 Feb 2024 12:22:23 -0600 Subject: [PATCH 054/114] Update squid_running_on_docker.sh (#2469) --- flavors/squid_auto/squid_running_on_docker.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/flavors/squid_auto/squid_running_on_docker.sh b/flavors/squid_auto/squid_running_on_docker.sh index 2d7cf8e688..812a9f7387 100644 --- a/flavors/squid_auto/squid_running_on_docker.sh +++ b/flavors/squid_auto/squid_running_on_docker.sh @@ -74,10 +74,18 @@ function install_docker(){ # Docker ############################################################### # Install docker from sources - curl -fsSL ${DOCKER_DOWNLOAD_URL}/gpg | sudo apt-key add - - add-apt-repository "deb [arch=amd64] ${DOCKER_DOWNLOAD_URL} $(lsb_release -cs) stable" - apt update - apt install -y docker-ce + if [[ $DISTRO == "Ubuntu" ]]; then + curl -fsSL ${DOCKER_DOWNLOAD_URL}/gpg | sudo apt-key add - + add-apt-repository "deb [arch=amd64] ${DOCKER_DOWNLOAD_URL} $(lsb_release -cs) stable" + apt update + apt install -y docker-ce + else + sudo yum update -y + sudo yum install -y docker + # Start and enable Docker service + sudo systemctl start docker + sudo systemctl enable docker + fi mkdir -p /etc/docker cp ${SUB_FOLDER}/flavors/squid_auto/startup_configs/docker-daemon.json /etc/docker/daemon.json chmod -R 0644 /etc/docker From 982e7b6fb1d7bf8fb1ba167e1adf2de7ba995a8b Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Thu, 15 Feb 2024 10:57:46 -0500 Subject: [PATCH 055/114] Lowering parallelism to 1 for VA workflows (#2470) * Lowering parallelism to 1 for VA workflows * Caught changes we don't want --- kube/services/argo/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml index 473f7041ea..7c2a04531f 100644 --- a/kube/services/argo/values.yaml +++ b/kube/services/argo/values.yaml @@ -1,5 +1,5 @@ controller: - parallelism: 3 + parallelism: 1 metricsConfig: # -- Enables prometheus metrics server enabled: true From 0015d1c83ba3e9d0f9cbda1d7eb774e8a57ee68a Mon Sep 17 00:00:00 2001 From: vzpgb <45467497+vzpgb@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:19:27 -0600 Subject: [PATCH 056/114] Increases VA prod parallel workflows to 3 (#2473) --- kube/services/argo/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml index 7c2a04531f..473f7041ea 100644 --- a/kube/services/argo/values.yaml +++ b/kube/services/argo/values.yaml @@ -1,5 +1,5 @@ controller: - parallelism: 1 + parallelism: 3 metricsConfig: # -- Enables prometheus metrics server enabled: true From f4e99c4c17f28bc4e4ad0bdd476f6b65b94ab183 Mon Sep 17 00:00:00 2001 From: burtonk <117617405+k-burt-uch@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:08:18 -0600 Subject: [PATCH 057/114] Add azure blob storage to web_wildcard_whitelist (#2475) --- packer/configs/web_wildcard_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/packer/configs/web_wildcard_whitelist b/packer/configs/web_wildcard_whitelist index c58eeefe80..621dec3d58 100644 --- a/packer/configs/web_wildcard_whitelist +++ b/packer/configs/web_wildcard_whitelist @@ -44,4 +44,5 @@ .yahooapis.com .cloudfront.net .docker.io +.blob.core.windows.net .googleapis.com From 1f8632ff493afad17e14d610eb35c01e1b81e359 Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Mon, 19 Feb 2024 16:12:31 -0600 Subject: [PATCH 058/114] Add whitelist for snap to work (#2476) --- files/squid_whitelist/web_whitelist | 1 - files/squid_whitelist/web_wildcard_whitelist | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index 625c20b299..c191b2e8c1 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -15,7 +15,6 @@ ctds-planx.atlassian.net data.cityofchicago.org dataguids.org api.login.yahoo.com -api.snapcraft.io apt.kubernetes.io argoproj.github.io archive.cloudera.com diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist index 3dca3946ad..b71ee76c20 100644 --- a/files/squid_whitelist/web_wildcard_whitelist +++ b/files/squid_whitelist/web_wildcard_whitelist @@ -97,6 +97,8 @@ .sks-keyservers.net .slack.com .slack-msgs.com +.snapcraft.io +.snapcraftcontent.com .sourceforge.net .southsideweekly.com .theanvil.io From fe5b5502603ec4158c9f077fad83eb421402763d Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:56:05 -0600 Subject: [PATCH 059/114] gen3 logs snapshot: also get initContainers logs (#2478) --- gen3/lib/logs/snapshot.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gen3/lib/logs/snapshot.sh b/gen3/lib/logs/snapshot.sh index 31cb80283b..d3d3b2c6cc 100644 --- a/gen3/lib/logs/snapshot.sh +++ b/gen3/lib/logs/snapshot.sh @@ -36,10 +36,11 @@ gen3_logs_snapshot_container() { # Snapshot all the pods # gen3_logs_snapshot_all() { + # For each pod for which we can list the containers, get the pod name and get its list of containers + # (container names + initContainers names). Diplay them as lines of " ". g3kubectl get pods -o json | \ - jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | map( {pod: .metadata.name, containers: .spec.containers | map(.name) } ) | map( .pod as $pod | .containers | map( { pod: $pod, cont: .})[]) | map(select(.cont != "pause" and .cont != "jupyterhub"))[] | .pod + " " + .cont' | \ + jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | map( {pod: .metadata.name, containers: [(.spec.containers | select(.!=null) | map(.name)), (.spec.initContainers | select(.!=null) | map(.name)) | add ] } ) | map( .pod as $pod | .containers | map( { pod: $pod, cont: .})[]) | map(select(.cont != "pause" and .cont != "jupyterhub"))[] | .pod + " " + .cont' | \ while read -r line; do gen3_logs_snapshot_container $line done } - From ff88b7b9ab6898587c96c78bb49ae6456cddabad Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:46:29 -0500 Subject: [PATCH 060/114] Adding namespace and overall parallelism so we can run 5 workflows in preprod, and 3 in prod at the same time (#2479) --- kube/services/argo/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml index 473f7041ea..2b46ced0f3 100644 --- a/kube/services/argo/values.yaml +++ b/kube/services/argo/values.yaml @@ -1,5 +1,6 @@ controller: - parallelism: 3 + parallelism: 8 + namespaceParallelism: 3 metricsConfig: # -- Enables prometheus metrics server enabled: true From d8fd2813a1e13e40bdb58d33abd2a6101f398bba Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Thu, 22 Feb 2024 10:47:16 -0600 Subject: [PATCH 061/114] feat(argo-wrapper): update mountpath (#2481) --- kube/services/argo-wrapper/argo-wrapper-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml index 00d118746c..89ec29ecce 100644 --- a/kube/services/argo-wrapper/argo-wrapper-deploy.yaml +++ b/kube/services/argo-wrapper/argo-wrapper-deploy.yaml @@ -75,5 +75,5 @@ spec: subPath: argo.json - name: argo-wrapper-namespace-config readOnly: true - mountPath: /src/config.ini + mountPath: /argowrapper/config.ini subPath: config.ini From 2df523111e44e3e1eb3ccc5563066ea1c13f4505 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Thu, 22 Feb 2024 11:03:28 -0600 Subject: [PATCH 062/114] feat(argo-wrapper): update config.ini, missed option (#2482) --- kube/services/argo-wrapper/config.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini index 334438ac20..1a1f20c291 100644 --- a/kube/services/argo-wrapper/config.ini +++ b/kube/services/argo-wrapper/config.ini @@ -2,3 +2,4 @@ ARGO_ACCESS_METHOD = access ARGO_HOST = $ARGO_HOST ARGO_NAMESPACE = $ARGO_NAMESPACE +COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={} From e51ed100e8b3b338b7029777a8daadf8b0e22002 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Thu, 22 Feb 2024 11:21:56 -0600 Subject: [PATCH 063/114] fix(argo-wrapper): fix typo (#2483) --- kube/services/argo-wrapper/config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini index 1a1f20c291..40ac392fd5 100644 --- a/kube/services/argo-wrapper/config.ini +++ b/kube/services/argo-wrapper/config.ini @@ -2,4 +2,4 @@ ARGO_ACCESS_METHOD = access ARGO_HOST = $ARGO_HOST ARGO_NAMESPACE = $ARGO_NAMESPACE -COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={} +COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={} From c5c54865cc2261bfd91b0d83cdbc50b8381254ef Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Fri, 23 Feb 2024 08:17:29 -0800 Subject: [PATCH 064/114] Prevent eviction of ETL job pod while running (#2484) --- kube/services/jobs/etl-job.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kube/services/jobs/etl-job.yaml b/kube/services/jobs/etl-job.yaml index fa201c99aa..6b9b887eca 100644 --- a/kube/services/jobs/etl-job.yaml +++ b/kube/services/jobs/etl-job.yaml @@ -2,6 +2,8 @@ apiVersion: batch/v1 kind: Job metadata: + annotations: + karpenter.sh/do-not-evict: "true" name: etl spec: backoffLimit: 0 From 13fb2397dfaac8165c7a71e1fce769bda08ee760 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Tue, 27 Feb 2024 11:51:18 -0500 Subject: [PATCH 065/114] Fix/argo wrapper default url (#2486) * Adding a secret to contain all values for any workflow templates * Adding an override namespace argument to kube-setup-argo * Fixing a typo * You'd think I'd never written Bash before * Adding the last part of the namespace override logic * Debug prints * Trying to figure out why the override namespace logic doesn't work * Just checking * Desperate * I think we can't use echo in gen3 scripts * Sanity check * Just a commit * Checking if we get the argo_namespace correctly * Making the print clearer * It's debug print time * Fixing references * Adding some stuff * Removing debugs, and adding --overwrite * Adding double quotes * Fixing kube-setup-argo-wrapper to have default values for argo-service URL and argo namespace * Fixing /tmp/config.ini collisions * Turns out that wasn't the way to go, instead we'll just delete /tmp/config.ini to account for multiple environments * Removed some extra stuff that snuck in --- .secrets.baseline | 2 +- gen3/bin/kube-setup-argo-wrapper.sh | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 0a8fe9cc99..936e306e55 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2023-10-26T21:32:44Z", + "generated_at": "2024-02-23T20:30:41Z", "plugins_used": [ { "name": "AWSKeyDetector" diff --git a/gen3/bin/kube-setup-argo-wrapper.sh b/gen3/bin/kube-setup-argo-wrapper.sh index 306050b124..9f7cc52ce4 100644 --- a/gen3/bin/kube-setup-argo-wrapper.sh +++ b/gen3/bin/kube-setup-argo-wrapper.sh @@ -18,17 +18,26 @@ if [[ -z "$GEN3_SOURCE_ONLY" ]]; then gen3 roll argo-wrapper g3kubectl apply -f "${GEN3_HOME}/kube/services/argo-wrapper/argo-wrapper-service.yaml" + if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then - argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url) + export ARGO_HOST=$(g3k_manifest_lookup .argo.argo_server_service_url) + else + export ARGO_HOST="http://argo-argo-workflows-server.argo.svc.cluster.local:2746" + fi + + if g3k_config_lookup '.argo_namespace' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json 2> /dev/null; then + export ARGO_NAMESPACE=$(g3k_config_lookup '.argo_namespace' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) + else + export ARGO_NAMESPACE="argo" + fi - export ARGO_HOST=${argo_server_service_url} - export ARGO_NAMESPACE=argo-$(gen3 db namespace) - envsubst <"${GEN3_HOME}/kube/services/argo-wrapper/config.ini" > /tmp/config.ini + envsubst <"${GEN3_HOME}/kube/services/argo-wrapper/config.ini" > /tmp/config.ini - g3kubectl delete configmap argo-wrapper-namespace-config - g3kubectl create configmap argo-wrapper-namespace-config --from-file /tmp/config.ini - fi + g3kubectl delete configmap argo-wrapper-namespace-config + g3kubectl create configmap argo-wrapper-namespace-config --from-file /tmp/config.ini + + rm /tmp/config.ini gen3_log_info "the argo-wrapper service has been deployed onto the kubernetes cluster" fi From 1a44bef0712ca5ffb4262615999787ef536644be Mon Sep 17 00:00:00 2001 From: emalinowski Date: Wed, 28 Feb 2024 06:20:07 -0600 Subject: [PATCH 066/114] fix(jenkins-zone): Added zone (#2487) Co-authored-by: Edward Malinowski Co-authored-by: Ajo Augustine --- kube/services/jenkins/jenkins-deploy.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kube/services/jenkins/jenkins-deploy.yaml b/kube/services/jenkins/jenkins-deploy.yaml index c0eae2040b..954e996f21 100644 --- a/kube/services/jenkins/jenkins-deploy.yaml +++ b/kube/services/jenkins/jenkins-deploy.yaml @@ -38,6 +38,10 @@ spec: operator: In values: - on-demand + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-1a serviceAccountName: jenkins-service securityContext: runAsUser: 1000 From 6c27fc9a7fe6cd3affd7a108db3c80b51c664ad1 Mon Sep 17 00:00:00 2001 From: Michael Lukowski Date: Thu, 29 Feb 2024 09:59:10 -0600 Subject: [PATCH 067/114] updating the cedar data ingest (#2472) * updating the cedar data ingest * fix unsanitized request * address some comments --- .../healdata/heal-cedar-data-ingest.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index 71575e3c56..d3ff246c34 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -35,6 +35,16 @@ "Buisness Development": "Business Development" } +# repository links +REPOSITORY_STUDY_ID_LINK_TEMPLATE = { + "NIDDK Central": "https://repository.niddk.nih.gov/studies//", + "NIDA Data Share": "https://datashare.nida.nih.gov/study/", + "NICHD DASH": "https://dash.nichd.nih.gov/study/", + "ICPSR": "https://www.icpsr.umich.edu/web/ICPSR/studies/", + "BioSystics-AP": "https://biosystics-ap.com/assays/assaystudy//", +} + + # Defines field that we don't want to include in the filters OMITTED_VALUES_MAPPING = { "study_metadata.human_subject_applicability.gender_applicability": "Not applicable" @@ -114,6 +124,31 @@ def get_client_token(client_id: str, client_secret: str): return token +def get_related_studies(serial_num, hostname): + related_study_result = [] + + if serial_num: + mds = requests.get(f"https://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000") + if mds.status_code == 200: + related_study_metadata = mds.json() + + for ( + related_study_metadata_key, + related_study_metadata_value, + ) in related_study_metadata.items(): + title = ( + related_study_metadata_value.get( + "gen3_discovery", {} + ) + .get("study_metadata", {}) + .get("minimal_info", {}) + .get("study_name", "") + ) + link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/" + related_study_result.append({"title": title, "link": link}) + return related_study_result + + parser = argparse.ArgumentParser() parser.add_argument("--directory", help="CEDAR Directory ID for registering ") @@ -214,6 +249,67 @@ def get_client_token(client_id: str, client_secret: str): mds_res["gen3_discovery"]["study_metadata"].update(cedar_record) mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites + # setup citations + doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "") + mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation + + + # setup repository_study_link + data_repositories = ( + mds_res.get("study_metadata", {}) + .get("metadata_location", {}) + .get("data_repositories", []) + ) + repository_citation = "Users must also include a citation to the data as specified by the local repository." + repository_citation_additional_text = ' The link to the study page at the local repository can be found in the "Data" tab.' + for repository in data_repositories: + if ( + repository["repository_name"] + and repository["repository_name"] + in REPOSITORY_STUDY_ID_LINK_TEMPLATE + and repository["repository_study_ID"] + ): + repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[ + repository["repository_name"] + ].replace("", repository["repository_study_ID"]) + repository.update({"repository_study_link": repository_study_link}) + if repository_citation_additional_text not in repository_citation: + repository_citation += repository_citation_additional_text + if len(data_repositories): + data_repositories[0] = { + **data_repositories[0], + "repository_citation": repository_citation, + } + mds_res["gen3_discovery"]["study_metadata"][ + "metadata_location" + ]["data_repositories"] = data_repositories + + + + # set up related studies + serial_num = None + try: + serial_num = ( + mds_res + .get("nih_reporter", {}) + .get("project_num_split", {}) + .get("serial_num", None) + ) + except Exception: + print(f"Unable to get serial number for study") + + if serial_num == None: + print(f"Unable to get serial number for study") + + related_study_result = get_related_studies(serial_num, hostname) + existing_related_study_result = mds_res.get("related_studies", []) + for related_study in related_study_result: + if related_study not in existing_related_study_result: + existing_related_study_result.append(copy.deepcopy(related_study)) + mds_res["gen3_discovery"][ + "related_studies" + ] = copy.deepcopy(existing_related_study_result) + # merge data from cedar that is not study level metadata into a level higher deleted_keys = [] for key, value in mds_res["gen3_discovery"]["study_metadata"].items(): From 072279d59732ff0109e9ca18bb05479b3bdcd577 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Thu, 29 Feb 2024 11:16:29 -0500 Subject: [PATCH 068/114] Feat/s3 bucket template (#2485) * Adding a secret to contain all values for any workflow templates * Adding an override namespace argument to kube-setup-argo * Fixing a typo * You'd think I'd never written Bash before * Adding the last part of the namespace override logic * Debug prints * Trying to figure out why the override namespace logic doesn't work * Just checking * Desperate * I think we can't use echo in gen3 scripts * Sanity check * Just a commit * Checking if we get the argo_namespace correctly * Making the print clearer * It's debug print time * Fixing references * Adding some stuff * Removing debugs, and adding --overwrite * Adding double quotes --- .secrets.baseline | 4 +-- gen3/bin/kube-setup-argo.sh | 66 ++++++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 936e306e55..b7e06622d6 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-02-23T20:30:41Z", + "generated_at": "2024-02-23T20:30:41Z" "plugins_used": [ { "name": "AWSKeyDetector" @@ -342,7 +342,7 @@ "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", "is_secret": false, "is_verified": false, - "line_number": 191, + "line_number": 206, "type": "Secret Keyword" } ], diff --git a/gen3/bin/kube-setup-argo.sh b/gen3/bin/kube-setup-argo.sh index ff2438833c..20676145b9 100644 --- a/gen3/bin/kube-setup-argo.sh +++ b/gen3/bin/kube-setup-argo.sh @@ -5,10 +5,25 @@ source "${GEN3_HOME}/gen3/lib/utils.sh" gen3_load "gen3/gen3setup" gen3_load "gen3/lib/kube-setup-init" +override_namespace=false +force=false + +for arg in "${@}"; do + if [ "$arg" == "--override-namespace" ]; then + override_namespace=true + elif [ "$arg" == "--force" ]; then + force=true + else + #Print usage info and exit + gen3_log_info "Usage: gen3 kube-setup-argo [--override-namespace] [--force]" + exit 1 + fi +done ctx="$(g3kubectl config current-context)" ctxNamespace="$(g3kubectl config view -ojson | jq -r ".contexts | map(select(.name==\"$ctx\")) | .[0] | .context.namespace")" +argo_namespace=$(g3k_config_lookup '.argo_namespace' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) function setup_argo_buckets { local accountNumber @@ -32,13 +47,13 @@ function setup_argo_buckets { roleName="gen3-argo-${environment//_/-}-role" bucketPolicy="argo-bucket-policy-${nameSpace}" internalBucketPolicy="argo-internal-bucket-policy-${nameSpace}" - if [[ ! -z $(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo."s3-bucket"') ]]; then - if [[ ! -z $(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) ]]; then + if [[ ! -z $(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo."downloadable-s3-bucket"') ]]; then + if [[ ! -z $(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) ]]; then gen3_log_info "Using S3 bucket found in manifest: ${bucketName}" - bucketName=$(g3k_config_lookup '."s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) + bucketName=$(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) else gen3_log_info "Using S3 bucket found in manifest: ${bucketName}" - bucketName=$(g3k_config_lookup '.argo."s3-bucket"') + bucketName=$(g3k_config_lookup '.argo."downloadable-s3-bucket"') fi fi if [[ ! -z $(g3k_config_lookup '."internal-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) || ! -z $(g3k_config_lookup '.argo."internal-s3-bucket"') ]]; then @@ -131,19 +146,19 @@ EOF g3kubectl create namespace argo || true g3kubectl label namespace argo app=argo || true # Grant admin access within the argo namespace to the default SA in the argo namespace - g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=argo:default -n argo || true + g3kubectl create rolebinding argo-admin --clusterrole=admin --serviceaccount=argo:default -n $argo_namespace || true fi gen3_log_info "Creating IAM role ${roleName}" if aws iam get-role --role-name "${roleName}" > /dev/null 2>&1; then gen3_log_info "IAM role ${roleName} already exists.." roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text) gen3_log_info "Role annotate" - g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n argo - g3kubectl annotate serviceaccount argo eks.amazonaws.com/role-arn=${roleArn} -n $nameSpace + g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $argo_namespace + g3kubectl annotate serviceaccount argo eks.amazonaws.com/role-arn=${roleArn} --overwrite -n $nameSpace else gen3 awsrole create $roleName argo $nameSpace -f all_namespaces roleArn=$(aws iam get-role --role-name "${roleName}" --query 'Role.Arn' --output text) - g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n argo + g3kubectl annotate serviceaccount default eks.amazonaws.com/role-arn=${roleArn} -n $argo_namespace fi # Grant admin access within the current namespace to the argo SA in the current namespace @@ -177,34 +192,47 @@ EOF for serviceName in indexd; do secretName="${serviceName}-creds" # Only delete if secret is found to prevent early exits - if [[ ! -z $(g3kubectl get secrets -n argo | grep $secretName) ]]; then - g3kubectl delete secret "$secretName" -n argo > /dev/null 2>&1 + if [[ ! -z $(g3kubectl get secrets -n $argo_namespace | grep $secretName) ]]; then + g3kubectl delete secret "$secretName" -n $argo_namespace > /dev/null 2>&1 fi done sleep 1 # I think delete is async - give backend a second to finish indexdFencePassword=$(cat $(gen3_secrets_folder)/creds.json | jq -r .indexd.user_db.$indexd_admin_user) - g3kubectl create secret generic "indexd-creds" --from-literal=user=$indexd_admin_user --from-literal=password=$indexdFencePassword -n argo + g3kubectl create secret generic "indexd-creds" --from-literal=user=$indexd_admin_user --from-literal=password=$indexdFencePassword -n $argo_namespace fi } function setup_argo_db() { - if ! secret="$(g3kubectl get secret argo-db-creds -n argo 2> /dev/null)"; then + if ! secret="$(g3kubectl get secret argo-db-creds -n $argo_namespace 2> /dev/null)"; then gen3_log_info "Setting up argo db persistence" gen3 db setup argo || true dbCreds=$(gen3 secrets decode argo-g3auto dbcreds.json) - g3kubectl create secret -n argo generic argo-db-creds --from-literal=db_host=$(echo $dbCreds | jq -r .db_host) --from-literal=db_username=$(echo $dbCreds | jq -r .db_username) --from-literal=db_password=$(echo $dbCreds | jq -r .db_password) --from-literal=db_database=$(echo $dbCreds | jq -r .db_database) + g3kubectl create secret -n $argo_namespace generic argo-db-creds --from-literal=db_host=$(echo $dbCreds | jq -r .db_host) --from-literal=db_username=$(echo $dbCreds | jq -r .db_username) --from-literal=db_password=$(echo $dbCreds | jq -r .db_password) --from-literal=db_database=$(echo $dbCreds | jq -r .db_database) else gen3_log_info "Argo DB setup already completed" fi } - setup_argo_buckets +function setup_argo_template_secret() { + gen3_log_info "Started the template secret process" + downloadable_bucket_name=$(g3k_config_lookup '."downloadable-s3-bucket"' $(g3k_manifest_init)/$(g3k_hostname)/manifests/argo/argo.json) + # Check if the secret already exists + if [[ ! -z $(g3kubectl get secret argo-template-values-secret -n $argo_namespace) ]]; then + gen3_log_info "Argo template values secret already exists, assuming it's stale and deleting" + g3kubectl delete secret argo-template-values-secret -n $argo_namespace + fi + gen3_log_info "Creating argo template values secret" + g3kubectl create secret generic argo-template-values-secret --from-literal=DOWNLOADABLE_BUCKET=$downloadable_bucket_name -n $argo_namespace +} + +setup_argo_buckets # only do this if we are running in the default namespace -if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" ]]; then +if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" || "$override_namespace" == true ]]; then setup_argo_db - if (! helm status argo -n argo > /dev/null 2>&1 ) || [[ "$1" == "--force" ]]; then - DBHOST=$(kubectl get secrets -n argo argo-db-creds -o json | jq -r .data.db_host | base64 -d) - DBNAME=$(kubectl get secrets -n argo argo-db-creds -o json | jq -r .data.db_database | base64 -d) + setup_argo_template_secret + if (! helm status argo -n $argo_namespace > /dev/null 2>&1 ) || [[ "$force" == true ]]; then + DBHOST=$(kubectl get secrets -n $argo_namespace argo-db-creds -o json | jq -r .data.db_host | base64 -d) + DBNAME=$(kubectl get secrets -n $argo_namespace argo-db-creds -o json | jq -r .data.db_database | base64 -d) if [[ -z $internalBucketName ]]; then BUCKET=$bucketName else @@ -218,7 +246,7 @@ if [[ "$ctxNamespace" == "default" || "$ctxNamespace" == "null" ]]; then helm repo add argo https://argoproj.github.io/argo-helm --force-update 2> >(grep -v 'This is insecure' >&2) helm repo update 2> >(grep -v 'This is insecure' >&2) - helm upgrade --install argo argo/argo-workflows -n argo -f ${valuesFile} --version 0.29.1 + helm upgrade --install argo argo/argo-workflows -n $argo_namespace -f ${valuesFile} --version 0.29.1 else gen3_log_info "kube-setup-argo exiting - argo already deployed, use --force to redeploy" fi From 90b66091c0d1d8b89e7182ad84201e99903d8e44 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:29:05 -0600 Subject: [PATCH 069/114] Update heal-cedar-data-ingest.py (#2490) --- files/scripts/healdata/heal-cedar-data-ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index d3ff246c34..1da4ac2d52 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -128,7 +128,7 @@ def get_related_studies(serial_num, hostname): related_study_result = [] if serial_num: - mds = requests.get(f"https://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000") + mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000") if mds.status_code == 200: related_study_metadata = mds.json() From bd6bc767c0461f8fc1f1d90fa8a755be1e7fc381 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:49:47 -0600 Subject: [PATCH 070/114] fix missed import (#2491) --- .secrets.baseline | 346 ++++-------------- .../healdata/heal-cedar-data-ingest.py | 2 +- 2 files changed, 73 insertions(+), 275 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index b7e06622d6..2583e269f8 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,9 +1,9 @@ { "exclude": { - "files": "^.secrets.baseline$", + "files": null, "lines": null }, - "generated_at": "2024-02-23T20:30:41Z" + "generated_at": "2024-02-29T19:38:46Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -61,14 +61,12 @@ "Chef/repo/data_bags/README.md": [ { "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366", - "is_secret": false, "is_verified": false, "line_number": 45, "type": "Secret Keyword" }, { "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee", - "is_secret": false, "is_verified": false, "line_number": 51, "type": "Secret Keyword" @@ -77,25 +75,22 @@ "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, - "line_number": 121, + "line_number": 124, "type": "Secret Keyword" } ], "Docker/jenkins/Jenkins-Worker/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, - "line_number": 143, + "line_number": 139, "type": "Secret Keyword" } ], "Docker/jenkins/Jenkins/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, "line_number": 107, "type": "Secret Keyword" @@ -104,7 +99,6 @@ "Docker/jenkins/Jenkins2/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, "line_number": 108, "type": "Secret Keyword" @@ -113,7 +107,6 @@ "Docker/sidecar/service.key": [ { "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", - "is_secret": false, "is_verified": false, "line_number": 1, "type": "Private Key" @@ -122,7 +115,6 @@ "Jenkins/Stacks/Jenkins/jenkins.env.sample": [ { "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Secret Keyword" @@ -131,7 +123,6 @@ "ansible/roles/awslogs/defaults/main.yaml": [ { "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Basic Auth Credentials" @@ -140,14 +131,12 @@ "ansible/roles/slurm/README.md": [ { "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42", - "is_secret": false, "is_verified": false, "line_number": 86, "type": "Base64 High Entropy String" }, { "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476", - "is_secret": false, "is_verified": false, "line_number": 86, "type": "Secret Keyword" @@ -156,7 +145,6 @@ "apis_configs/config_helper.py": [ { "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", - "is_secret": false, "is_verified": false, "line_number": 66, "type": "Basic Auth Credentials" @@ -165,7 +153,6 @@ "apis_configs/fence_credentials.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 23, "type": "Secret Keyword" @@ -174,21 +161,18 @@ "apis_configs/fence_settings.py": [ { "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Basic Auth Credentials" }, { "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3", - "is_secret": false, "is_verified": false, "line_number": 58, "type": "Secret Keyword" }, { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", - "is_secret": false, "is_verified": false, "line_number": 80, "type": "Basic Auth Credentials" @@ -197,7 +181,6 @@ "apis_configs/indexd_settings.py": [ { "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc", - "is_secret": false, "is_verified": false, "line_number": 59, "type": "Basic Auth Credentials" @@ -206,7 +189,6 @@ "apis_configs/peregrine_settings.py": [ { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", - "is_secret": false, "is_verified": false, "line_number": 46, "type": "Basic Auth Credentials" @@ -215,7 +197,6 @@ "apis_configs/sheepdog_settings.py": [ { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", - "is_secret": false, "is_verified": false, "line_number": 46, "type": "Basic Auth Credentials" @@ -224,7 +205,6 @@ "doc/Gen3-data-upload.md": [ { "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" @@ -233,7 +213,6 @@ "doc/api.md": [ { "hashed_secret": "625de83a7517422051911680cc803921ff99db90", - "is_secret": false, "is_verified": false, "line_number": 47, "type": "Hex High Entropy String" @@ -242,28 +221,24 @@ "doc/gen3OnK8s.md": [ { "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069", - "is_secret": false, "is_verified": false, "line_number": 113, "type": "Secret Keyword" }, { "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2", - "is_secret": false, "is_verified": false, "line_number": 143, "type": "Secret Keyword" }, { "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48", - "is_secret": false, "is_verified": false, "line_number": 170, "type": "Secret Keyword" }, { "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce", - "is_secret": false, "is_verified": false, "line_number": 189, "type": "Secret Keyword" @@ -272,7 +247,6 @@ "doc/kube-setup-data-ingestion-job.md": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Secret Keyword" @@ -281,7 +255,6 @@ "doc/logs.md": [ { "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Secret Keyword" @@ -290,7 +263,6 @@ "doc/slurm_cluster.md": [ { "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", - "is_secret": false, "is_verified": false, "line_number": 184, "type": "Secret Keyword" @@ -299,14 +271,12 @@ "files/dashboard/usage-reports/package-lock.json": [ { "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448", - "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" }, { "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b", - "is_secret": false, "is_verified": false, "line_number": 15, "type": "Base64 High Entropy String" @@ -315,14 +285,12 @@ "gen3/bin/api.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 407, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", - "is_secret": false, "is_verified": false, "line_number": 477, "type": "Secret Keyword" @@ -331,7 +299,6 @@ "gen3/bin/kube-dev-namespace.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 135, "type": "Secret Keyword" @@ -340,7 +307,6 @@ "gen3/bin/kube-setup-argo.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 206, "type": "Secret Keyword" @@ -349,7 +315,6 @@ "gen3/bin/kube-setup-aurora-monitoring.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 59, "type": "Secret Keyword" @@ -358,7 +323,6 @@ "gen3/bin/kube-setup-certs.sh": [ { "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99", - "is_secret": false, "is_verified": false, "line_number": 50, "type": "Secret Keyword" @@ -367,14 +331,12 @@ "gen3/bin/kube-setup-dashboard.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 40, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", - "is_secret": false, "is_verified": false, "line_number": 41, "type": "Secret Keyword" @@ -383,14 +345,12 @@ "gen3/bin/kube-setup-data-ingestion-job.sh": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 37, "type": "Secret Keyword" }, { "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c", - "is_secret": false, "is_verified": false, "line_number": 102, "type": "Secret Keyword" @@ -399,7 +359,6 @@ "gen3/bin/kube-setup-dicom-server.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 43, "type": "Secret Keyword" @@ -408,7 +367,6 @@ "gen3/bin/kube-setup-dicom.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -417,14 +375,26 @@ "gen3/bin/kube-setup-jenkins.sh": [ { "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, + "is_verified": false, + "line_number": 22, + "type": "Secret Keyword" + } + ], + "gen3/bin/kube-setup-jenkins2.sh": [ + { + "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", + "is_verified": false, + "line_number": 18, + "type": "Secret Keyword" + }, + { + "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", "is_verified": false, "line_number": 22, "type": "Secret Keyword" @@ -433,7 +403,6 @@ "gen3/bin/kube-setup-metadata.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 35, "type": "Secret Keyword" @@ -442,21 +411,18 @@ "gen3/bin/kube-setup-revproxy.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 38, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 55, "type": "Secret Keyword" }, { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 57, "type": "Secret Keyword" @@ -465,21 +431,18 @@ "gen3/bin/kube-setup-secrets.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 79, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 82, "type": "Secret Keyword" }, { "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8", - "is_secret": false, "is_verified": false, "line_number": 95, "type": "Secret Keyword" @@ -488,14 +451,12 @@ "gen3/bin/kube-setup-sftp.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 36, "type": "Secret Keyword" }, { "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4", - "is_secret": false, "is_verified": false, "line_number": 51, "type": "Secret Keyword" @@ -504,7 +465,6 @@ "gen3/bin/kube-setup-sheepdog.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 33, "type": "Secret Keyword" @@ -513,28 +473,24 @@ "gen3/bin/kube-setup-sower-jobs.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 25, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 120, "type": "Secret Keyword" }, { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 122, "type": "Secret Keyword" @@ -543,21 +499,18 @@ "gen3/bin/kube-setup-ssjdispatcher.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 117, "type": "Secret Keyword" }, { "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93", - "is_secret": false, "is_verified": false, "line_number": 184, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 197, "type": "Secret Keyword" @@ -566,14 +519,12 @@ "gen3/lib/aws.sh": [ { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", - "is_secret": false, "is_verified": false, "line_number": 640, "type": "Secret Keyword" }, { "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8", - "is_secret": false, "is_verified": false, "line_number": 660, "type": "Secret Keyword" @@ -582,14 +533,12 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [ { "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", - "is_secret": false, "is_verified": false, "line_number": 33, "type": "Basic Auth Credentials" }, { "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db", - "is_secret": false, "is_verified": false, "line_number": 286, "type": "Secret Keyword" @@ -598,7 +547,6 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" @@ -607,7 +555,6 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Secret Keyword" @@ -616,7 +563,6 @@ "gen3/lib/logs/utils.sh": [ { "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5", - "is_secret": false, "is_verified": false, "line_number": 3, "type": "Secret Keyword" @@ -625,7 +571,6 @@ "gen3/lib/manifestDefaults/hatchery/hatchery.json": [ { "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039", - "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -634,14 +579,12 @@ "gen3/lib/onprem.sh": [ { "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3", - "is_secret": false, "is_verified": false, "line_number": 68, "type": "Secret Keyword" }, { "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70", - "is_secret": false, "is_verified": false, "line_number": 84, "type": "Secret Keyword" @@ -650,14 +593,12 @@ "gen3/lib/secrets/rotate-postgres.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 162, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 250, "type": "Secret Keyword" @@ -666,49 +607,42 @@ "gen3/lib/testData/etlconvert/expected2.yaml": [ { "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474", - "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" }, { "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55", - "is_secret": false, "is_verified": false, "line_number": 13, "type": "Base64 High Entropy String" }, { "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26", - "is_secret": false, "is_verified": false, "line_number": 16, "type": "Base64 High Entropy String" }, { "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424", - "is_secret": false, "is_verified": false, "line_number": 33, "type": "Base64 High Entropy String" }, { "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba", - "is_secret": false, "is_verified": false, "line_number": 35, "type": "Base64 High Entropy String" }, { "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22", - "is_secret": false, "is_verified": false, "line_number": 36, "type": "Base64 High Entropy String" @@ -717,7 +651,6 @@ "gen3/test/secretsTest.sh": [ { "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938", - "is_secret": false, "is_verified": false, "line_number": 25, "type": "Secret Keyword" @@ -726,28 +659,24 @@ "gen3/test/terraformTest.sh": [ { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", - "is_secret": false, "is_verified": false, "line_number": 156, "type": "Secret Keyword" }, { "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009", - "is_secret": false, "is_verified": false, "line_number": 172, "type": "Base64 High Entropy String" }, { "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef", - "is_secret": false, "is_verified": false, "line_number": 175, "type": "Base64 High Entropy String" }, { "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c", - "is_secret": false, "is_verified": false, "line_number": 175, "type": "Secret Keyword" @@ -756,21 +685,18 @@ "kube/services/argocd/values.yaml": [ { "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd", - "is_secret": false, "is_verified": false, "line_number": 360, "type": "Private Key" }, { "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61", - "is_secret": false, "is_verified": false, "line_number": 379, "type": "Secret Keyword" }, { "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2", - "is_secret": false, "is_verified": false, "line_number": 412, "type": "Secret Keyword" @@ -779,7 +705,6 @@ "kube/services/datadog/values.yaml": [ { "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b", - "is_secret": false, "is_verified": false, "line_number": 23, "type": "Secret Keyword" @@ -788,401 +713,362 @@ "kube/services/fenceshib/fenceshib-configmap.yaml": [ { "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c", - "is_secret": false, "is_verified": false, "line_number": 375, "type": "Base64 High Entropy String" }, { "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f", - "is_secret": false, "is_verified": false, "line_number": 376, "type": "Base64 High Entropy String" }, { "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b", - "is_secret": false, "is_verified": false, "line_number": 377, "type": "Base64 High Entropy String" }, { "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5", - "is_secret": false, "is_verified": false, "line_number": 378, "type": "Base64 High Entropy String" }, { "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a", - "is_secret": false, "is_verified": false, "line_number": 379, "type": "Base64 High Entropy String" }, { "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9", - "is_secret": false, "is_verified": false, "line_number": 380, "type": "Base64 High Entropy String" }, { "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12", - "is_secret": false, "is_verified": false, "line_number": 381, "type": "Base64 High Entropy String" }, { "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7", - "is_secret": false, "is_verified": false, "line_number": 382, "type": "Base64 High Entropy String" }, { "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a", - "is_secret": false, "is_verified": false, "line_number": 383, "type": "Base64 High Entropy String" }, { "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3", - "is_secret": false, "is_verified": false, "line_number": 384, "type": "Base64 High Entropy String" }, { "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c", - "is_secret": false, "is_verified": false, "line_number": 385, "type": "Base64 High Entropy String" }, { "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52", - "is_secret": false, "is_verified": false, "line_number": 386, "type": "Base64 High Entropy String" }, { "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110", - "is_secret": false, "is_verified": false, "line_number": 387, "type": "Base64 High Entropy String" }, { "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7", - "is_secret": false, "is_verified": false, "line_number": 388, "type": "Base64 High Entropy String" }, { "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e", - "is_secret": false, "is_verified": false, "line_number": 389, "type": "Base64 High Entropy String" }, { "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1", - "is_secret": false, "is_verified": false, "line_number": 390, "type": "Base64 High Entropy String" }, { "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53", - "is_secret": false, "is_verified": false, "line_number": 391, "type": "Base64 High Entropy String" }, { "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a", - "is_secret": false, "is_verified": false, "line_number": 392, "type": "Base64 High Entropy String" }, { "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294", - "is_secret": false, "is_verified": false, "line_number": 393, "type": "Base64 High Entropy String" }, { "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41", - "is_secret": false, "is_verified": false, "line_number": 394, "type": "Base64 High Entropy String" }, { "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1", - "is_secret": false, "is_verified": false, "line_number": 395, "type": "Base64 High Entropy String" }, { "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb", - "is_secret": false, "is_verified": false, "line_number": 396, "type": "Base64 High Entropy String" }, { "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433", - "is_secret": false, "is_verified": false, "line_number": 397, "type": "Base64 High Entropy String" }, { "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a", - "is_secret": false, "is_verified": false, "line_number": 398, "type": "Base64 High Entropy String" }, { "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2", - "is_secret": false, "is_verified": false, "line_number": 399, "type": "Base64 High Entropy String" }, { - "hashed_secret": "7803ae08cdc22a5e0b025eff3c9ef0628eedc165", - "is_secret": false, + "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525", "is_verified": false, "line_number": 419, "type": "Base64 High Entropy String" }, { - "hashed_secret": "b8b61e87f5b58b0eeb597b2122ea0cea2ccab3d9", - "is_secret": false, + "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab", "is_verified": false, "line_number": 420, "type": "Base64 High Entropy String" }, { - "hashed_secret": "787745fc904c3bd7eddc3d1aab683a376c13890f", - "is_secret": false, + "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf", "is_verified": false, "line_number": 423, "type": "Base64 High Entropy String" }, { - "hashed_secret": "81361d672f238f505a6246ef9b655ee2f48d67e7", - "is_secret": false, + "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8", "is_verified": false, "line_number": 424, "type": "Base64 High Entropy String" }, { - "hashed_secret": "7c98bff76ac3f273d15ed9bc3dd5294d323ab577", - "is_secret": false, + "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2", "is_verified": false, "line_number": 425, "type": "Base64 High Entropy String" }, { - "hashed_secret": "46038fc88daceed8dd46817ca45c72ae0270fdd4", - "is_secret": false, + "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa", "is_verified": false, "line_number": 426, "type": "Base64 High Entropy String" }, { - "hashed_secret": "acad0c57b4f5cbed1b4863ed06d02784180a9f92", - "is_secret": false, + "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b", "is_verified": false, "line_number": 427, "type": "Base64 High Entropy String" }, { - "hashed_secret": "1b57f49a6ee337c16ecd6aabfc0dff3b3821cd09", - "is_secret": false, + "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f", "is_verified": false, "line_number": 428, "type": "Base64 High Entropy String" }, { - "hashed_secret": "5b688158be36e8b3f265a462ed599dcf69290084", - "is_secret": false, + "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48", "is_verified": false, "line_number": 429, "type": "Base64 High Entropy String" }, { - "hashed_secret": "965996e12c8b50b3c325d96003e8984a4ece658a", - "is_secret": false, + "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f", "is_verified": false, "line_number": 430, "type": "Base64 High Entropy String" }, { - "hashed_secret": "584f0c58e764e948af1a35c9e60447aa0f84c6f5", - "is_secret": false, + "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e", "is_verified": false, "line_number": 431, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bcaf897786d060a675ee9d654a84ae8baf96e9d0", - "is_secret": false, + "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46", "is_verified": false, "line_number": 432, "type": "Base64 High Entropy String" }, { - "hashed_secret": "0c09277fa183e06d32065f9386a3b4190b445df3", - "is_secret": false, + "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8", "is_verified": false, "line_number": 433, "type": "Base64 High Entropy String" }, { - "hashed_secret": "5a51be06b305d6664e4afd25f21869b0f8b5039b", - "is_secret": false, + "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d", "is_verified": false, "line_number": 434, "type": "Base64 High Entropy String" }, { - "hashed_secret": "b38404f8853d734e3d03577b2c1084b4540c8708", - "is_secret": false, + "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15", "is_verified": false, "line_number": 435, "type": "Base64 High Entropy String" }, { - "hashed_secret": "126ccc602cffcb8292beb57137f7f6719e317b72", - "is_secret": false, + "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f", "is_verified": false, "line_number": 436, "type": "Base64 High Entropy String" }, { - "hashed_secret": "6681c1d7e1d327642a32cb8864ad51e4b8f981e5", - "is_secret": false, + "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811", "is_verified": false, "line_number": 437, "type": "Base64 High Entropy String" }, { - "hashed_secret": "7f7b1f316ece195e5f584fe2faf6f9edc6942c6f", - "is_secret": false, + "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1", + "is_verified": false, + "line_number": 438, + "type": "Base64 High Entropy String" + }, + { + "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8", "is_verified": false, "line_number": 439, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bb908c7bc655057f2edc42815c5dff82e9dea529", - "is_secret": false, + "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f", "is_verified": false, "line_number": 440, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bc2a0d18e3dd142df7b34e95342d47bf8aadabcb", - "is_secret": false, + "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa", "is_verified": false, "line_number": 441, "type": "Base64 High Entropy String" }, { - "hashed_secret": "d60f0bcea109bb6edb6e45fd387f5f2c86e49e1a", - "is_secret": false, + "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870", "is_verified": false, "line_number": 442, "type": "Base64 High Entropy String" }, { - "hashed_secret": "e549dd40a741557cc1c4e377df0a141354e22688", - "is_secret": false, + "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383", "is_verified": false, "line_number": 443, "type": "Base64 High Entropy String" }, { - "hashed_secret": "2dd2486dae84cad50387c20bf687b6fbc6162b58", - "is_secret": false, + "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7", "is_verified": false, "line_number": 444, "type": "Base64 High Entropy String" }, { - "hashed_secret": "71622010fc7eb09d9273f59c548bde6a5da5dc0e", - "is_secret": false, + "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8", "is_verified": false, "line_number": 445, "type": "Base64 High Entropy String" }, { - "hashed_secret": "6f0115cf53bd49ec990c562ac6cbfc452c83cd46", - "is_secret": false, + "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106", "is_verified": false, "line_number": 446, "type": "Base64 High Entropy String" }, { - "hashed_secret": "70dddd534b2f9bb70871fefe0845b79c3b69363f", - "is_secret": false, + "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554", + "is_verified": false, + "line_number": 447, + "type": "Base64 High Entropy String" + }, + { + "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413", "is_verified": false, "line_number": 448, "type": "Base64 High Entropy String" }, { - "hashed_secret": "acf3536b0416aa99608b0be17e87655370ece829", - "is_secret": false, + "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1", + "is_verified": false, + "line_number": 451, + "type": "Base64 High Entropy String" + }, + { + "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5", "is_verified": false, - "line_number": 449, + "line_number": 452, "type": "Base64 High Entropy String" }, { - "hashed_secret": "1d13ee35c7279c1fae1c6474ed47611994273e41", - "is_secret": false, + "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963", "is_verified": false, - "line_number": 450, + "line_number": 453, "type": "Base64 High Entropy String" }, { - "hashed_secret": "d38cf89b25bd7378cdb4e00b4b59293001dd500b", - "is_secret": false, + "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591", "is_verified": false, - "line_number": 451, + "line_number": 454, "type": "Base64 High Entropy String" }, { - "hashed_secret": "1648f34ce2f1b563a8ed1c6d5d55b5e76a395903", - "is_secret": false, + "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911", "is_verified": false, - "line_number": 452, + "line_number": 455, "type": "Base64 High Entropy String" }, { - "hashed_secret": "9bf63f6f49fb01ff80959bc5a60c8688df92cc02", - "is_secret": false, + "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a", "is_verified": false, - "line_number": 453, + "line_number": 456, "type": "Base64 High Entropy String" } ], "kube/services/jobs/indexd-authz-job.yaml": [ { "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", - "is_secret": false, "is_verified": false, "line_number": 87, "type": "Basic Auth Credentials" @@ -1191,14 +1077,12 @@ "kube/services/monitoring/grafana-values.yaml": [ { "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385", - "is_secret": false, "is_verified": false, "line_number": 162, "type": "Secret Keyword" }, { "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4", - "is_secret": false, "is_verified": false, "line_number": 166, "type": "Secret Keyword" @@ -1207,7 +1091,6 @@ "kube/services/revproxy/helpers.js": [ { "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af", - "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" @@ -1216,7 +1099,6 @@ "kube/services/revproxy/helpersTest.js": [ { "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb", - "is_secret": false, "is_verified": false, "line_number": 22, "type": "JSON Web Token" @@ -1225,7 +1107,6 @@ "kube/services/superset/superset-deploy.yaml": [ { "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7", - "is_secret": false, "is_verified": false, "line_number": 38, "type": "Secret Keyword" @@ -1234,14 +1115,12 @@ "kube/services/superset/superset-redis.yaml": [ { "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f", - "is_secret": false, "is_verified": false, "line_number": 165, "type": "Secret Keyword" }, { "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9", - "is_secret": false, "is_verified": false, "line_number": 265, "type": "Secret Keyword" @@ -1250,35 +1129,30 @@ "kube/services/superset/values.yaml": [ { "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3", - "is_secret": false, "is_verified": false, "line_number": 54, "type": "Secret Keyword" }, { "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46", - "is_secret": false, "is_verified": false, "line_number": 86, "type": "Secret Keyword" }, { "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8", - "is_secret": false, "is_verified": false, "line_number": 212, "type": "Secret Keyword" }, { "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358", - "is_secret": false, "is_verified": false, "line_number": 396, "type": "Secret Keyword" }, { "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc", - "is_secret": false, "is_verified": false, "line_number": 503, "type": "Secret Keyword" @@ -1287,280 +1161,240 @@ "package-lock.json": [ { "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d", - "is_secret": false, "is_verified": false, "line_number": 1481, "type": "Base64 High Entropy String" }, { "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87", - "is_secret": false, "is_verified": false, "line_number": 1489, "type": "Base64 High Entropy String" }, { "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00", - "is_secret": false, "is_verified": false, "line_number": 1521, "type": "Base64 High Entropy String" }, { "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04", - "is_secret": false, "is_verified": false, "line_number": 1547, "type": "Base64 High Entropy String" }, { "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af", - "is_secret": false, "is_verified": false, "line_number": 1611, "type": "Base64 High Entropy String" }, { "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93", - "is_secret": false, "is_verified": false, "line_number": 1640, "type": "Base64 High Entropy String" }, { "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e", - "is_secret": false, "is_verified": false, "line_number": 1648, "type": "Base64 High Entropy String" }, { "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e", - "is_secret": false, "is_verified": false, "line_number": 1664, "type": "Base64 High Entropy String" }, { "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e", - "is_secret": false, "is_verified": false, "line_number": 1683, "type": "Base64 High Entropy String" }, { "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56", - "is_secret": false, "is_verified": false, "line_number": 1733, "type": "Base64 High Entropy String" }, { "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3", - "is_secret": false, "is_verified": false, "line_number": 1742, "type": "Base64 High Entropy String" }, { "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e", - "is_secret": false, "is_verified": false, "line_number": 1755, "type": "Base64 High Entropy String" }, { "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85", - "is_secret": false, "is_verified": false, "line_number": 1769, "type": "Base64 High Entropy String" }, { "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72", - "is_secret": false, "is_verified": false, "line_number": 1855, "type": "Base64 High Entropy String" }, { "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5", - "is_secret": false, "is_verified": false, "line_number": 1889, "type": "Base64 High Entropy String" }, { "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c", - "is_secret": false, "is_verified": false, "line_number": 1894, "type": "Base64 High Entropy String" }, { "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c", - "is_secret": false, "is_verified": false, "line_number": 1921, "type": "Base64 High Entropy String" }, { "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620", - "is_secret": false, "is_verified": false, "line_number": 1950, "type": "Base64 High Entropy String" }, { "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71", - "is_secret": false, "is_verified": false, "line_number": 1983, "type": "Base64 High Entropy String" }, { "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d", - "is_secret": false, "is_verified": false, "line_number": 2004, "type": "Base64 High Entropy String" }, { "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6", - "is_secret": false, "is_verified": false, "line_number": 2013, "type": "Base64 High Entropy String" }, { "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b", - "is_secret": false, "is_verified": false, "line_number": 2046, "type": "Base64 High Entropy String" }, { "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466", - "is_secret": false, "is_verified": false, "line_number": 2078, "type": "Base64 High Entropy String" }, { "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f", - "is_secret": false, "is_verified": false, "line_number": 2083, "type": "Base64 High Entropy String" }, { "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265", - "is_secret": false, "is_verified": false, "line_number": 2111, "type": "Base64 High Entropy String" }, { "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc", - "is_secret": false, "is_verified": false, "line_number": 2138, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759", - "is_secret": false, "is_verified": false, "line_number": 2143, "type": "Base64 High Entropy String" }, { "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523", - "is_secret": false, "is_verified": false, "line_number": 2150, "type": "Base64 High Entropy String" }, { "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430", - "is_secret": false, "is_verified": false, "line_number": 2188, "type": "Base64 High Entropy String" }, { "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8", - "is_secret": false, "is_verified": false, "line_number": 2291, "type": "Base64 High Entropy String" }, { "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f", - "is_secret": false, "is_verified": false, "line_number": 2303, "type": "Base64 High Entropy String" }, { "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672", - "is_secret": false, "is_verified": false, "line_number": 2308, "type": "Base64 High Entropy String" }, { "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539", - "is_secret": false, "is_verified": false, "line_number": 2317, "type": "Base64 High Entropy String" }, { "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0", - "is_secret": false, "is_verified": false, "line_number": 2327, "type": "Base64 High Entropy String" }, { "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671", - "is_secret": false, "is_verified": false, "line_number": 2332, "type": "Base64 High Entropy String" }, { "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97", - "is_secret": false, "is_verified": false, "line_number": 2340, "type": "Base64 High Entropy String" }, { "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd", - "is_secret": false, "is_verified": false, "line_number": 2349, "type": "Base64 High Entropy String" }, { "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14", - "is_secret": false, "is_verified": false, "line_number": 2357, "type": "Base64 High Entropy String" }, { "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471", - "is_secret": false, "is_verified": false, "line_number": 2369, "type": "Base64 High Entropy String" }, { "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136", - "is_secret": false, "is_verified": false, "line_number": 2387, "type": "Base64 High Entropy String" @@ -1569,7 +1403,6 @@ "tf_files/aws/cognito/README.md": [ { "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70", - "is_secret": false, "is_verified": false, "line_number": 106, "type": "Secret Keyword" @@ -1578,14 +1411,12 @@ "tf_files/aws/commons/README.md": [ { "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed", - "is_secret": false, "is_verified": false, "line_number": 60, "type": "Secret Keyword" }, { "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e", - "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -1594,7 +1425,6 @@ "tf_files/aws/eks/sample.tfvars": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", - "is_secret": false, "is_verified": false, "line_number": 107, "type": "Hex High Entropy String" @@ -1603,7 +1433,6 @@ "tf_files/aws/eks/variables.tf": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", - "is_secret": false, "is_verified": false, "line_number": 133, "type": "Hex High Entropy String" @@ -1612,14 +1441,12 @@ "tf_files/aws/modules/common-logging/README.md": [ { "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", - "is_secret": false, "is_verified": false, "line_number": 57, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", - "is_secret": false, "is_verified": false, "line_number": 59, "type": "Hex High Entropy String" @@ -1628,28 +1455,24 @@ "tf_files/aws/modules/common-logging/lambda_function.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Hex High Entropy String" @@ -1658,21 +1481,18 @@ "tf_files/aws/modules/common-logging/testLambda.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" @@ -1681,7 +1501,6 @@ "tf_files/aws/modules/eks/variables.tf": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", - "is_secret": false, "is_verified": false, "line_number": 113, "type": "Hex High Entropy String" @@ -1690,14 +1509,12 @@ "tf_files/aws/modules/management-logs/README.md": [ { "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", - "is_secret": false, "is_verified": false, "line_number": 54, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", - "is_secret": false, "is_verified": false, "line_number": 56, "type": "Hex High Entropy String" @@ -1706,28 +1523,24 @@ "tf_files/aws/modules/management-logs/lambda_function.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Hex High Entropy String" @@ -1736,42 +1549,36 @@ "tf_files/aws/modules/management-logs/testLambda.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Base64 High Entropy String" }, { "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Hex High Entropy String" }, { "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Hex High Entropy String" @@ -1780,7 +1587,6 @@ "tf_files/aws/slurm/README.md": [ { "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d", - "is_secret": false, "is_verified": false, "line_number": 83, "type": "Secret Keyword" @@ -1789,7 +1595,6 @@ "tf_files/azure/cloud.tf": [ { "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4", - "is_secret": false, "is_verified": false, "line_number": 424, "type": "Secret Keyword" @@ -1798,7 +1603,6 @@ "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", - "is_secret": false, "is_verified": false, "line_number": 231, "type": "Secret Keyword" @@ -1807,7 +1611,6 @@ "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", - "is_secret": false, "is_verified": false, "line_number": 231, "type": "Secret Keyword" @@ -1816,7 +1619,6 @@ "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", - "is_secret": false, "is_verified": false, "line_number": 262, "type": "Secret Keyword" @@ -1825,21 +1627,18 @@ "tf_files/gcp/commons/sample.tfvars": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 11, "type": "Secret Keyword" }, { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" }, { "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105", - "is_secret": false, "is_verified": false, "line_number": 37, "type": "Secret Keyword" @@ -1848,7 +1647,6 @@ "tf_files/shared/modules/k8s_configs/creds.tpl": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 8, "type": "Secret Keyword" diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index 1da4ac2d52..730a3b36e8 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -1,5 +1,5 @@ import argparse -import json +import copy import sys import requests import pydash From 8103e82b1044daca4290831db63653b40b24b97c Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Thu, 29 Feb 2024 12:58:32 -0800 Subject: [PATCH 071/114] Add jenkins-dcp to the env pool reset script (#2489) --- files/scripts/ci-env-pool-reset.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/files/scripts/ci-env-pool-reset.sh b/files/scripts/ci-env-pool-reset.sh index c0c1f67c6d..362cfbfd57 100644 --- a/files/scripts/ci-env-pool-reset.sh +++ b/files/scripts/ci-env-pool-reset.sh @@ -29,6 +29,7 @@ source "${GEN3_HOME}/gen3/gen3setup.sh" cat - > jenkins-envs-services.txt < Date: Thu, 29 Feb 2024 15:20:15 -0800 Subject: [PATCH 072/114] (HP-572) create dynamodb table for gen3-license workspace (#2441) * (HP-572) create dynamodb table for gen3-license workspace * (HP-572) get GSI from hatchery config * (HP-572) remove extra quotes from 'GSI' * (HP-572) move distribute-license for backwards compatibility --------- Co-authored-by: Mingfei Shao <2475897+mfshao@users.noreply.github.com> --- gen3/bin/kube-setup-hatchery.sh | 57 ++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index 691fb354ab..5454d1e248 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -20,11 +20,60 @@ gen3 jupyter j-namespace setup # (g3k_kv_filter ${GEN3_HOME}/kube/services/hatchery/serviceaccount.yaml BINDING_ONE "name: hatchery-binding1-$namespace" BINDING_TWO "name: hatchery-binding2-$namespace" CURRENT_NAMESPACE "namespace: $namespace" | g3kubectl apply -f -) || true +function exists_or_create_gen3_license_table() { + # Create dynamodb table for gen3-license if it does not exist. + TARGET_TABLE="$1" + echo "Checking for dynamoDB table: ${TARGET_TABLE}" -# cron job to distribute licenses if using Stata workspaces -if [ "$(g3kubectl get configmaps/manifest-hatchery -o yaml | grep "\"image\": .*stata.*")" ]; -then - gen3 job cron distribute-licenses '* * * * *' + FOUND_TABLE=`aws dynamodb list-tables | jq -r .TableNames | jq -c -r '.[]' | grep $TARGET_TABLE` + if [ -n "$FOUND_TABLE" ]; then + echo "Target table already exists in dynamoDB: $FOUND_TABLE" + else + echo "Creating table ${TARGET_TABLE}" + GSI=`g3kubectl get configmaps/manifest-hatchery -o json | jq -r '.data."license-user-maps-global-secondary-index"'` + if [[ -z "$GSI" || "$GSI" == "null" ]]; then + echo "Error: No global-secondary-index in configuration" + return 0 + fi + aws dynamodb create-table \ + --no-cli-pager \ + --table-name "$TARGET_TABLE" \ + --attribute-definitions AttributeName=itemId,AttributeType=S \ + AttributeName=environment,AttributeType=S \ + AttributeName=isActive,AttributeType=S \ + --key-schema AttributeName=itemId,KeyType=HASH \ + AttributeName=environment,KeyType=RANGE \ + --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \ + --global-secondary-indexes \ + "[ + { + \"IndexName\": \"$GSI\", + \"KeySchema\": [{\"AttributeName\":\"environment\",\"KeyType\":\"HASH\"}, + {\"AttributeName\":\"isActive\",\"KeyType\":\"RANGE\"}], + \"Projection\":{ + \"ProjectionType\":\"INCLUDE\", + \"NonKeyAttributes\":[\"itemId\",\"userId\",\"licenseId\",\"licenseType\"] + }, + \"ProvisionedThroughput\": { + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 3 + } + } + ]" + fi +} + +TARGET_TABLE=`g3kubectl get configmaps/manifest-hatchery -o json | jq -r '.data."license-user-maps-dynamodb-table"'` +if [[ -z "$TARGET_TABLE" || "$TARGET_TABLE" == "null" ]]; then + echo "No gen3-license table in configuration" + # cron job to distribute licenses if using Stata workspaces but not using dynamoDB + if [ "$(g3kubectl get configmaps/manifest-hatchery -o yaml | grep "\"image\": .*stata.*")" ]; + then + gen3 job cron distribute-licenses '* * * * *' + fi +else + echo "Found gen3-license table in configuration: $TARGET_TABLE" + exists_or_create_gen3_license_table "$TARGET_TABLE" fi policy=$( cat < Date: Fri, 1 Mar 2024 15:46:10 -0600 Subject: [PATCH 073/114] Fix/revert (#2493) * Revert "fix missed import (#2491)" This reverts commit bd6bc767c0461f8fc1f1d90fa8a755be1e7fc381. * Revert "Update heal-cedar-data-ingest.py (#2490)" This reverts commit 90b66091c0d1d8b89e7182ad84201e99903d8e44. * Revert "updating the cedar data ingest (#2472)" This reverts commit 6c27fc9a7fe6cd3affd7a108db3c80b51c664ad1. --- .secrets.baseline | 346 ++++++++++++++---- .../healdata/heal-cedar-data-ingest.py | 98 +---- 2 files changed, 275 insertions(+), 169 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 2583e269f8..b7e06622d6 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,9 +1,9 @@ { "exclude": { - "files": null, + "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-02-29T19:38:46Z", + "generated_at": "2024-02-23T20:30:41Z" "plugins_used": [ { "name": "AWSKeyDetector" @@ -61,12 +61,14 @@ "Chef/repo/data_bags/README.md": [ { "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366", + "is_secret": false, "is_verified": false, "line_number": 45, "type": "Secret Keyword" }, { "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee", + "is_secret": false, "is_verified": false, "line_number": 51, "type": "Secret Keyword" @@ -75,22 +77,25 @@ "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "is_secret": false, "is_verified": false, - "line_number": 124, + "line_number": 121, "type": "Secret Keyword" } ], "Docker/jenkins/Jenkins-Worker/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "is_secret": false, "is_verified": false, - "line_number": 139, + "line_number": 143, "type": "Secret Keyword" } ], "Docker/jenkins/Jenkins/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "is_secret": false, "is_verified": false, "line_number": 107, "type": "Secret Keyword" @@ -99,6 +104,7 @@ "Docker/jenkins/Jenkins2/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "is_secret": false, "is_verified": false, "line_number": 108, "type": "Secret Keyword" @@ -107,6 +113,7 @@ "Docker/sidecar/service.key": [ { "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", + "is_secret": false, "is_verified": false, "line_number": 1, "type": "Private Key" @@ -115,6 +122,7 @@ "Jenkins/Stacks/Jenkins/jenkins.env.sample": [ { "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Secret Keyword" @@ -123,6 +131,7 @@ "ansible/roles/awslogs/defaults/main.yaml": [ { "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", + "is_secret": false, "is_verified": false, "line_number": 30, "type": "Basic Auth Credentials" @@ -131,12 +140,14 @@ "ansible/roles/slurm/README.md": [ { "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42", + "is_secret": false, "is_verified": false, "line_number": 86, "type": "Base64 High Entropy String" }, { "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476", + "is_secret": false, "is_verified": false, "line_number": 86, "type": "Secret Keyword" @@ -145,6 +156,7 @@ "apis_configs/config_helper.py": [ { "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", + "is_secret": false, "is_verified": false, "line_number": 66, "type": "Basic Auth Credentials" @@ -153,6 +165,7 @@ "apis_configs/fence_credentials.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": false, "is_verified": false, "line_number": 23, "type": "Secret Keyword" @@ -161,18 +174,21 @@ "apis_configs/fence_settings.py": [ { "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf", + "is_secret": false, "is_verified": false, "line_number": 6, "type": "Basic Auth Credentials" }, { "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3", + "is_secret": false, "is_verified": false, "line_number": 58, "type": "Secret Keyword" }, { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "is_secret": false, "is_verified": false, "line_number": 80, "type": "Basic Auth Credentials" @@ -181,6 +197,7 @@ "apis_configs/indexd_settings.py": [ { "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc", + "is_secret": false, "is_verified": false, "line_number": 59, "type": "Basic Auth Credentials" @@ -189,6 +206,7 @@ "apis_configs/peregrine_settings.py": [ { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "is_secret": false, "is_verified": false, "line_number": 46, "type": "Basic Auth Credentials" @@ -197,6 +215,7 @@ "apis_configs/sheepdog_settings.py": [ { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "is_secret": false, "is_verified": false, "line_number": 46, "type": "Basic Auth Credentials" @@ -205,6 +224,7 @@ "doc/Gen3-data-upload.md": [ { "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e", + "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" @@ -213,6 +233,7 @@ "doc/api.md": [ { "hashed_secret": "625de83a7517422051911680cc803921ff99db90", + "is_secret": false, "is_verified": false, "line_number": 47, "type": "Hex High Entropy String" @@ -221,24 +242,28 @@ "doc/gen3OnK8s.md": [ { "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069", + "is_secret": false, "is_verified": false, "line_number": 113, "type": "Secret Keyword" }, { "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2", + "is_secret": false, "is_verified": false, "line_number": 143, "type": "Secret Keyword" }, { "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48", + "is_secret": false, "is_verified": false, "line_number": 170, "type": "Secret Keyword" }, { "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce", + "is_secret": false, "is_verified": false, "line_number": 189, "type": "Secret Keyword" @@ -247,6 +272,7 @@ "doc/kube-setup-data-ingestion-job.md": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": false, "is_verified": false, "line_number": 30, "type": "Secret Keyword" @@ -255,6 +281,7 @@ "doc/logs.md": [ { "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f", + "is_secret": false, "is_verified": false, "line_number": 6, "type": "Secret Keyword" @@ -263,6 +290,7 @@ "doc/slurm_cluster.md": [ { "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", + "is_secret": false, "is_verified": false, "line_number": 184, "type": "Secret Keyword" @@ -271,12 +299,14 @@ "files/dashboard/usage-reports/package-lock.json": [ { "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448", + "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" }, { "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b", + "is_secret": false, "is_verified": false, "line_number": 15, "type": "Base64 High Entropy String" @@ -285,12 +315,14 @@ "gen3/bin/api.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 407, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", + "is_secret": false, "is_verified": false, "line_number": 477, "type": "Secret Keyword" @@ -299,6 +331,7 @@ "gen3/bin/kube-dev-namespace.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 135, "type": "Secret Keyword" @@ -307,6 +340,7 @@ "gen3/bin/kube-setup-argo.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "is_secret": false, "is_verified": false, "line_number": 206, "type": "Secret Keyword" @@ -315,6 +349,7 @@ "gen3/bin/kube-setup-aurora-monitoring.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 59, "type": "Secret Keyword" @@ -323,6 +358,7 @@ "gen3/bin/kube-setup-certs.sh": [ { "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99", + "is_secret": false, "is_verified": false, "line_number": 50, "type": "Secret Keyword" @@ -331,12 +367,14 @@ "gen3/bin/kube-setup-dashboard.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "is_secret": false, "is_verified": false, "line_number": 40, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", + "is_secret": false, "is_verified": false, "line_number": 41, "type": "Secret Keyword" @@ -345,12 +383,14 @@ "gen3/bin/kube-setup-data-ingestion-job.sh": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": false, "is_verified": false, "line_number": 37, "type": "Secret Keyword" }, { "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c", + "is_secret": false, "is_verified": false, "line_number": 102, "type": "Secret Keyword" @@ -359,6 +399,7 @@ "gen3/bin/kube-setup-dicom-server.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 43, "type": "Secret Keyword" @@ -367,6 +408,7 @@ "gen3/bin/kube-setup-dicom.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -375,26 +417,14 @@ "gen3/bin/kube-setup-jenkins.sh": [ { "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_verified": false, - "line_number": 22, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-setup-jenkins2.sh": [ - { - "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", - "is_verified": false, - "line_number": 18, - "type": "Secret Keyword" - }, - { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 22, "type": "Secret Keyword" @@ -403,6 +433,7 @@ "gen3/bin/kube-setup-metadata.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 35, "type": "Secret Keyword" @@ -411,18 +442,21 @@ "gen3/bin/kube-setup-revproxy.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "is_secret": false, "is_verified": false, "line_number": 38, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 55, "type": "Secret Keyword" }, { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 57, "type": "Secret Keyword" @@ -431,18 +465,21 @@ "gen3/bin/kube-setup-secrets.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 79, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 82, "type": "Secret Keyword" }, { "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8", + "is_secret": false, "is_verified": false, "line_number": 95, "type": "Secret Keyword" @@ -451,12 +488,14 @@ "gen3/bin/kube-setup-sftp.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 36, "type": "Secret Keyword" }, { "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4", + "is_secret": false, "is_verified": false, "line_number": 51, "type": "Secret Keyword" @@ -465,6 +504,7 @@ "gen3/bin/kube-setup-sheepdog.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 33, "type": "Secret Keyword" @@ -473,24 +513,28 @@ "gen3/bin/kube-setup-sower-jobs.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "is_secret": false, "is_verified": false, "line_number": 25, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", + "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 120, "type": "Secret Keyword" }, { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 122, "type": "Secret Keyword" @@ -499,18 +543,21 @@ "gen3/bin/kube-setup-ssjdispatcher.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 117, "type": "Secret Keyword" }, { "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93", + "is_secret": false, "is_verified": false, "line_number": 184, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 197, "type": "Secret Keyword" @@ -519,12 +566,14 @@ "gen3/lib/aws.sh": [ { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", + "is_secret": false, "is_verified": false, "line_number": 640, "type": "Secret Keyword" }, { "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8", + "is_secret": false, "is_verified": false, "line_number": 660, "type": "Secret Keyword" @@ -533,12 +582,14 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [ { "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", + "is_secret": false, "is_verified": false, "line_number": 33, "type": "Basic Auth Credentials" }, { "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db", + "is_secret": false, "is_verified": false, "line_number": 286, "type": "Secret Keyword" @@ -547,6 +598,7 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" @@ -555,6 +607,7 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Secret Keyword" @@ -563,6 +616,7 @@ "gen3/lib/logs/utils.sh": [ { "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5", + "is_secret": false, "is_verified": false, "line_number": 3, "type": "Secret Keyword" @@ -571,6 +625,7 @@ "gen3/lib/manifestDefaults/hatchery/hatchery.json": [ { "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039", + "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -579,12 +634,14 @@ "gen3/lib/onprem.sh": [ { "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3", + "is_secret": false, "is_verified": false, "line_number": 68, "type": "Secret Keyword" }, { "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70", + "is_secret": false, "is_verified": false, "line_number": 84, "type": "Secret Keyword" @@ -593,12 +650,14 @@ "gen3/lib/secrets/rotate-postgres.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 162, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "is_secret": false, "is_verified": false, "line_number": 250, "type": "Secret Keyword" @@ -607,42 +666,49 @@ "gen3/lib/testData/etlconvert/expected2.yaml": [ { "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474", + "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" }, { "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55", + "is_secret": false, "is_verified": false, "line_number": 13, "type": "Base64 High Entropy String" }, { "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26", + "is_secret": false, "is_verified": false, "line_number": 16, "type": "Base64 High Entropy String" }, { "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424", + "is_secret": false, "is_verified": false, "line_number": 33, "type": "Base64 High Entropy String" }, { "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba", + "is_secret": false, "is_verified": false, "line_number": 35, "type": "Base64 High Entropy String" }, { "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22", + "is_secret": false, "is_verified": false, "line_number": 36, "type": "Base64 High Entropy String" @@ -651,6 +717,7 @@ "gen3/test/secretsTest.sh": [ { "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938", + "is_secret": false, "is_verified": false, "line_number": 25, "type": "Secret Keyword" @@ -659,24 +726,28 @@ "gen3/test/terraformTest.sh": [ { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", + "is_secret": false, "is_verified": false, "line_number": 156, "type": "Secret Keyword" }, { "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009", + "is_secret": false, "is_verified": false, "line_number": 172, "type": "Base64 High Entropy String" }, { "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef", + "is_secret": false, "is_verified": false, "line_number": 175, "type": "Base64 High Entropy String" }, { "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c", + "is_secret": false, "is_verified": false, "line_number": 175, "type": "Secret Keyword" @@ -685,18 +756,21 @@ "kube/services/argocd/values.yaml": [ { "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd", + "is_secret": false, "is_verified": false, "line_number": 360, "type": "Private Key" }, { "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61", + "is_secret": false, "is_verified": false, "line_number": 379, "type": "Secret Keyword" }, { "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2", + "is_secret": false, "is_verified": false, "line_number": 412, "type": "Secret Keyword" @@ -705,6 +779,7 @@ "kube/services/datadog/values.yaml": [ { "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b", + "is_secret": false, "is_verified": false, "line_number": 23, "type": "Secret Keyword" @@ -713,362 +788,401 @@ "kube/services/fenceshib/fenceshib-configmap.yaml": [ { "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c", + "is_secret": false, "is_verified": false, "line_number": 375, "type": "Base64 High Entropy String" }, { "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f", + "is_secret": false, "is_verified": false, "line_number": 376, "type": "Base64 High Entropy String" }, { "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b", + "is_secret": false, "is_verified": false, "line_number": 377, "type": "Base64 High Entropy String" }, { "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5", + "is_secret": false, "is_verified": false, "line_number": 378, "type": "Base64 High Entropy String" }, { "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a", + "is_secret": false, "is_verified": false, "line_number": 379, "type": "Base64 High Entropy String" }, { "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9", + "is_secret": false, "is_verified": false, "line_number": 380, "type": "Base64 High Entropy String" }, { "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12", + "is_secret": false, "is_verified": false, "line_number": 381, "type": "Base64 High Entropy String" }, { "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7", + "is_secret": false, "is_verified": false, "line_number": 382, "type": "Base64 High Entropy String" }, { "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a", + "is_secret": false, "is_verified": false, "line_number": 383, "type": "Base64 High Entropy String" }, { "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3", + "is_secret": false, "is_verified": false, "line_number": 384, "type": "Base64 High Entropy String" }, { "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c", + "is_secret": false, "is_verified": false, "line_number": 385, "type": "Base64 High Entropy String" }, { "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52", + "is_secret": false, "is_verified": false, "line_number": 386, "type": "Base64 High Entropy String" }, { "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110", + "is_secret": false, "is_verified": false, "line_number": 387, "type": "Base64 High Entropy String" }, { "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7", + "is_secret": false, "is_verified": false, "line_number": 388, "type": "Base64 High Entropy String" }, { "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e", + "is_secret": false, "is_verified": false, "line_number": 389, "type": "Base64 High Entropy String" }, { "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1", + "is_secret": false, "is_verified": false, "line_number": 390, "type": "Base64 High Entropy String" }, { "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53", + "is_secret": false, "is_verified": false, "line_number": 391, "type": "Base64 High Entropy String" }, { "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a", + "is_secret": false, "is_verified": false, "line_number": 392, "type": "Base64 High Entropy String" }, { "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294", + "is_secret": false, "is_verified": false, "line_number": 393, "type": "Base64 High Entropy String" }, { "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41", + "is_secret": false, "is_verified": false, "line_number": 394, "type": "Base64 High Entropy String" }, { "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1", + "is_secret": false, "is_verified": false, "line_number": 395, "type": "Base64 High Entropy String" }, { "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb", + "is_secret": false, "is_verified": false, "line_number": 396, "type": "Base64 High Entropy String" }, { "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433", + "is_secret": false, "is_verified": false, "line_number": 397, "type": "Base64 High Entropy String" }, { "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a", + "is_secret": false, "is_verified": false, "line_number": 398, "type": "Base64 High Entropy String" }, { "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2", + "is_secret": false, "is_verified": false, "line_number": 399, "type": "Base64 High Entropy String" }, { - "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525", + "hashed_secret": "7803ae08cdc22a5e0b025eff3c9ef0628eedc165", + "is_secret": false, "is_verified": false, "line_number": 419, "type": "Base64 High Entropy String" }, { - "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab", + "hashed_secret": "b8b61e87f5b58b0eeb597b2122ea0cea2ccab3d9", + "is_secret": false, "is_verified": false, "line_number": 420, "type": "Base64 High Entropy String" }, { - "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf", + "hashed_secret": "787745fc904c3bd7eddc3d1aab683a376c13890f", + "is_secret": false, "is_verified": false, "line_number": 423, "type": "Base64 High Entropy String" }, { - "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8", + "hashed_secret": "81361d672f238f505a6246ef9b655ee2f48d67e7", + "is_secret": false, "is_verified": false, "line_number": 424, "type": "Base64 High Entropy String" }, { - "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2", + "hashed_secret": "7c98bff76ac3f273d15ed9bc3dd5294d323ab577", + "is_secret": false, "is_verified": false, "line_number": 425, "type": "Base64 High Entropy String" }, { - "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa", + "hashed_secret": "46038fc88daceed8dd46817ca45c72ae0270fdd4", + "is_secret": false, "is_verified": false, "line_number": 426, "type": "Base64 High Entropy String" }, { - "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b", + "hashed_secret": "acad0c57b4f5cbed1b4863ed06d02784180a9f92", + "is_secret": false, "is_verified": false, "line_number": 427, "type": "Base64 High Entropy String" }, { - "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f", + "hashed_secret": "1b57f49a6ee337c16ecd6aabfc0dff3b3821cd09", + "is_secret": false, "is_verified": false, "line_number": 428, "type": "Base64 High Entropy String" }, { - "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48", + "hashed_secret": "5b688158be36e8b3f265a462ed599dcf69290084", + "is_secret": false, "is_verified": false, "line_number": 429, "type": "Base64 High Entropy String" }, { - "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f", + "hashed_secret": "965996e12c8b50b3c325d96003e8984a4ece658a", + "is_secret": false, "is_verified": false, "line_number": 430, "type": "Base64 High Entropy String" }, { - "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e", + "hashed_secret": "584f0c58e764e948af1a35c9e60447aa0f84c6f5", + "is_secret": false, "is_verified": false, "line_number": 431, "type": "Base64 High Entropy String" }, { - "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46", + "hashed_secret": "bcaf897786d060a675ee9d654a84ae8baf96e9d0", + "is_secret": false, "is_verified": false, "line_number": 432, "type": "Base64 High Entropy String" }, { - "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8", + "hashed_secret": "0c09277fa183e06d32065f9386a3b4190b445df3", + "is_secret": false, "is_verified": false, "line_number": 433, "type": "Base64 High Entropy String" }, { - "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d", + "hashed_secret": "5a51be06b305d6664e4afd25f21869b0f8b5039b", + "is_secret": false, "is_verified": false, "line_number": 434, "type": "Base64 High Entropy String" }, { - "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15", + "hashed_secret": "b38404f8853d734e3d03577b2c1084b4540c8708", + "is_secret": false, "is_verified": false, "line_number": 435, "type": "Base64 High Entropy String" }, { - "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f", + "hashed_secret": "126ccc602cffcb8292beb57137f7f6719e317b72", + "is_secret": false, "is_verified": false, "line_number": 436, "type": "Base64 High Entropy String" }, { - "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811", + "hashed_secret": "6681c1d7e1d327642a32cb8864ad51e4b8f981e5", + "is_secret": false, "is_verified": false, "line_number": 437, "type": "Base64 High Entropy String" }, { - "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1", - "is_verified": false, - "line_number": 438, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8", + "hashed_secret": "7f7b1f316ece195e5f584fe2faf6f9edc6942c6f", + "is_secret": false, "is_verified": false, "line_number": 439, "type": "Base64 High Entropy String" }, { - "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f", + "hashed_secret": "bb908c7bc655057f2edc42815c5dff82e9dea529", + "is_secret": false, "is_verified": false, "line_number": 440, "type": "Base64 High Entropy String" }, { - "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa", + "hashed_secret": "bc2a0d18e3dd142df7b34e95342d47bf8aadabcb", + "is_secret": false, "is_verified": false, "line_number": 441, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870", + "hashed_secret": "d60f0bcea109bb6edb6e45fd387f5f2c86e49e1a", + "is_secret": false, "is_verified": false, "line_number": 442, "type": "Base64 High Entropy String" }, { - "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383", + "hashed_secret": "e549dd40a741557cc1c4e377df0a141354e22688", + "is_secret": false, "is_verified": false, "line_number": 443, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7", + "hashed_secret": "2dd2486dae84cad50387c20bf687b6fbc6162b58", + "is_secret": false, "is_verified": false, "line_number": 444, "type": "Base64 High Entropy String" }, { - "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8", + "hashed_secret": "71622010fc7eb09d9273f59c548bde6a5da5dc0e", + "is_secret": false, "is_verified": false, "line_number": 445, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106", + "hashed_secret": "6f0115cf53bd49ec990c562ac6cbfc452c83cd46", + "is_secret": false, "is_verified": false, "line_number": 446, "type": "Base64 High Entropy String" }, { - "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554", - "is_verified": false, - "line_number": 447, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413", + "hashed_secret": "70dddd534b2f9bb70871fefe0845b79c3b69363f", + "is_secret": false, "is_verified": false, "line_number": 448, "type": "Base64 High Entropy String" }, { - "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1", - "is_verified": false, - "line_number": 451, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5", + "hashed_secret": "acf3536b0416aa99608b0be17e87655370ece829", + "is_secret": false, "is_verified": false, - "line_number": 452, + "line_number": 449, "type": "Base64 High Entropy String" }, { - "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963", + "hashed_secret": "1d13ee35c7279c1fae1c6474ed47611994273e41", + "is_secret": false, "is_verified": false, - "line_number": 453, + "line_number": 450, "type": "Base64 High Entropy String" }, { - "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591", + "hashed_secret": "d38cf89b25bd7378cdb4e00b4b59293001dd500b", + "is_secret": false, "is_verified": false, - "line_number": 454, + "line_number": 451, "type": "Base64 High Entropy String" }, { - "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911", + "hashed_secret": "1648f34ce2f1b563a8ed1c6d5d55b5e76a395903", + "is_secret": false, "is_verified": false, - "line_number": 455, + "line_number": 452, "type": "Base64 High Entropy String" }, { - "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a", + "hashed_secret": "9bf63f6f49fb01ff80959bc5a60c8688df92cc02", + "is_secret": false, "is_verified": false, - "line_number": 456, + "line_number": 453, "type": "Base64 High Entropy String" } ], "kube/services/jobs/indexd-authz-job.yaml": [ { "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", + "is_secret": false, "is_verified": false, "line_number": 87, "type": "Basic Auth Credentials" @@ -1077,12 +1191,14 @@ "kube/services/monitoring/grafana-values.yaml": [ { "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385", + "is_secret": false, "is_verified": false, "line_number": 162, "type": "Secret Keyword" }, { "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4", + "is_secret": false, "is_verified": false, "line_number": 166, "type": "Secret Keyword" @@ -1091,6 +1207,7 @@ "kube/services/revproxy/helpers.js": [ { "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af", + "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" @@ -1099,6 +1216,7 @@ "kube/services/revproxy/helpersTest.js": [ { "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb", + "is_secret": false, "is_verified": false, "line_number": 22, "type": "JSON Web Token" @@ -1107,6 +1225,7 @@ "kube/services/superset/superset-deploy.yaml": [ { "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7", + "is_secret": false, "is_verified": false, "line_number": 38, "type": "Secret Keyword" @@ -1115,12 +1234,14 @@ "kube/services/superset/superset-redis.yaml": [ { "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f", + "is_secret": false, "is_verified": false, "line_number": 165, "type": "Secret Keyword" }, { "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9", + "is_secret": false, "is_verified": false, "line_number": 265, "type": "Secret Keyword" @@ -1129,30 +1250,35 @@ "kube/services/superset/values.yaml": [ { "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3", + "is_secret": false, "is_verified": false, "line_number": 54, "type": "Secret Keyword" }, { "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46", + "is_secret": false, "is_verified": false, "line_number": 86, "type": "Secret Keyword" }, { "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8", + "is_secret": false, "is_verified": false, "line_number": 212, "type": "Secret Keyword" }, { "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358", + "is_secret": false, "is_verified": false, "line_number": 396, "type": "Secret Keyword" }, { "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc", + "is_secret": false, "is_verified": false, "line_number": 503, "type": "Secret Keyword" @@ -1161,240 +1287,280 @@ "package-lock.json": [ { "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d", + "is_secret": false, "is_verified": false, "line_number": 1481, "type": "Base64 High Entropy String" }, { "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87", + "is_secret": false, "is_verified": false, "line_number": 1489, "type": "Base64 High Entropy String" }, { "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00", + "is_secret": false, "is_verified": false, "line_number": 1521, "type": "Base64 High Entropy String" }, { "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04", + "is_secret": false, "is_verified": false, "line_number": 1547, "type": "Base64 High Entropy String" }, { "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af", + "is_secret": false, "is_verified": false, "line_number": 1611, "type": "Base64 High Entropy String" }, { "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93", + "is_secret": false, "is_verified": false, "line_number": 1640, "type": "Base64 High Entropy String" }, { "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e", + "is_secret": false, "is_verified": false, "line_number": 1648, "type": "Base64 High Entropy String" }, { "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e", + "is_secret": false, "is_verified": false, "line_number": 1664, "type": "Base64 High Entropy String" }, { "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e", + "is_secret": false, "is_verified": false, "line_number": 1683, "type": "Base64 High Entropy String" }, { "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56", + "is_secret": false, "is_verified": false, "line_number": 1733, "type": "Base64 High Entropy String" }, { "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3", + "is_secret": false, "is_verified": false, "line_number": 1742, "type": "Base64 High Entropy String" }, { "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e", + "is_secret": false, "is_verified": false, "line_number": 1755, "type": "Base64 High Entropy String" }, { "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85", + "is_secret": false, "is_verified": false, "line_number": 1769, "type": "Base64 High Entropy String" }, { "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72", + "is_secret": false, "is_verified": false, "line_number": 1855, "type": "Base64 High Entropy String" }, { "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5", + "is_secret": false, "is_verified": false, "line_number": 1889, "type": "Base64 High Entropy String" }, { "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c", + "is_secret": false, "is_verified": false, "line_number": 1894, "type": "Base64 High Entropy String" }, { "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c", + "is_secret": false, "is_verified": false, "line_number": 1921, "type": "Base64 High Entropy String" }, { "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620", + "is_secret": false, "is_verified": false, "line_number": 1950, "type": "Base64 High Entropy String" }, { "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71", + "is_secret": false, "is_verified": false, "line_number": 1983, "type": "Base64 High Entropy String" }, { "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d", + "is_secret": false, "is_verified": false, "line_number": 2004, "type": "Base64 High Entropy String" }, { "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6", + "is_secret": false, "is_verified": false, "line_number": 2013, "type": "Base64 High Entropy String" }, { "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b", + "is_secret": false, "is_verified": false, "line_number": 2046, "type": "Base64 High Entropy String" }, { "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466", + "is_secret": false, "is_verified": false, "line_number": 2078, "type": "Base64 High Entropy String" }, { "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f", + "is_secret": false, "is_verified": false, "line_number": 2083, "type": "Base64 High Entropy String" }, { "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265", + "is_secret": false, "is_verified": false, "line_number": 2111, "type": "Base64 High Entropy String" }, { "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc", + "is_secret": false, "is_verified": false, "line_number": 2138, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759", + "is_secret": false, "is_verified": false, "line_number": 2143, "type": "Base64 High Entropy String" }, { "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523", + "is_secret": false, "is_verified": false, "line_number": 2150, "type": "Base64 High Entropy String" }, { "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430", + "is_secret": false, "is_verified": false, "line_number": 2188, "type": "Base64 High Entropy String" }, { "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8", + "is_secret": false, "is_verified": false, "line_number": 2291, "type": "Base64 High Entropy String" }, { "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f", + "is_secret": false, "is_verified": false, "line_number": 2303, "type": "Base64 High Entropy String" }, { "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672", + "is_secret": false, "is_verified": false, "line_number": 2308, "type": "Base64 High Entropy String" }, { "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539", + "is_secret": false, "is_verified": false, "line_number": 2317, "type": "Base64 High Entropy String" }, { "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0", + "is_secret": false, "is_verified": false, "line_number": 2327, "type": "Base64 High Entropy String" }, { "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671", + "is_secret": false, "is_verified": false, "line_number": 2332, "type": "Base64 High Entropy String" }, { "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97", + "is_secret": false, "is_verified": false, "line_number": 2340, "type": "Base64 High Entropy String" }, { "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd", + "is_secret": false, "is_verified": false, "line_number": 2349, "type": "Base64 High Entropy String" }, { "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14", + "is_secret": false, "is_verified": false, "line_number": 2357, "type": "Base64 High Entropy String" }, { "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471", + "is_secret": false, "is_verified": false, "line_number": 2369, "type": "Base64 High Entropy String" }, { "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136", + "is_secret": false, "is_verified": false, "line_number": 2387, "type": "Base64 High Entropy String" @@ -1403,6 +1569,7 @@ "tf_files/aws/cognito/README.md": [ { "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70", + "is_secret": false, "is_verified": false, "line_number": 106, "type": "Secret Keyword" @@ -1411,12 +1578,14 @@ "tf_files/aws/commons/README.md": [ { "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed", + "is_secret": false, "is_verified": false, "line_number": 60, "type": "Secret Keyword" }, { "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e", + "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -1425,6 +1594,7 @@ "tf_files/aws/eks/sample.tfvars": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", + "is_secret": false, "is_verified": false, "line_number": 107, "type": "Hex High Entropy String" @@ -1433,6 +1603,7 @@ "tf_files/aws/eks/variables.tf": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", + "is_secret": false, "is_verified": false, "line_number": 133, "type": "Hex High Entropy String" @@ -1441,12 +1612,14 @@ "tf_files/aws/modules/common-logging/README.md": [ { "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", + "is_secret": false, "is_verified": false, "line_number": 57, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", + "is_secret": false, "is_verified": false, "line_number": 59, "type": "Hex High Entropy String" @@ -1455,24 +1628,28 @@ "tf_files/aws/modules/common-logging/lambda_function.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", + "is_secret": false, "is_verified": false, "line_number": 30, "type": "Hex High Entropy String" @@ -1481,18 +1658,21 @@ "tf_files/aws/modules/common-logging/testLambda.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" @@ -1501,6 +1681,7 @@ "tf_files/aws/modules/eks/variables.tf": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", + "is_secret": false, "is_verified": false, "line_number": 113, "type": "Hex High Entropy String" @@ -1509,12 +1690,14 @@ "tf_files/aws/modules/management-logs/README.md": [ { "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", + "is_secret": false, "is_verified": false, "line_number": 54, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", + "is_secret": false, "is_verified": false, "line_number": 56, "type": "Hex High Entropy String" @@ -1523,24 +1706,28 @@ "tf_files/aws/modules/management-logs/lambda_function.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", + "is_secret": false, "is_verified": false, "line_number": 30, "type": "Hex High Entropy String" @@ -1549,36 +1736,42 @@ "tf_files/aws/modules/management-logs/testLambda.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a", + "is_secret": false, "is_verified": false, "line_number": 6, "type": "Base64 High Entropy String" }, { "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d", + "is_secret": false, "is_verified": false, "line_number": 6, "type": "Hex High Entropy String" }, { "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4", + "is_secret": false, "is_verified": false, "line_number": 6, "type": "Hex High Entropy String" @@ -1587,6 +1780,7 @@ "tf_files/aws/slurm/README.md": [ { "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d", + "is_secret": false, "is_verified": false, "line_number": 83, "type": "Secret Keyword" @@ -1595,6 +1789,7 @@ "tf_files/azure/cloud.tf": [ { "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4", + "is_secret": false, "is_verified": false, "line_number": 424, "type": "Secret Keyword" @@ -1603,6 +1798,7 @@ "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", + "is_secret": false, "is_verified": false, "line_number": 231, "type": "Secret Keyword" @@ -1611,6 +1807,7 @@ "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", + "is_secret": false, "is_verified": false, "line_number": 231, "type": "Secret Keyword" @@ -1619,6 +1816,7 @@ "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", + "is_secret": false, "is_verified": false, "line_number": 262, "type": "Secret Keyword" @@ -1627,18 +1825,21 @@ "tf_files/gcp/commons/sample.tfvars": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_secret": false, "is_verified": false, "line_number": 11, "type": "Secret Keyword" }, { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", + "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" }, { "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105", + "is_secret": false, "is_verified": false, "line_number": 37, "type": "Secret Keyword" @@ -1647,6 +1848,7 @@ "tf_files/shared/modules/k8s_configs/creds.tpl": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "is_secret": false, "is_verified": false, "line_number": 8, "type": "Secret Keyword" diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index 730a3b36e8..71575e3c56 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -1,5 +1,5 @@ import argparse -import copy +import json import sys import requests import pydash @@ -35,16 +35,6 @@ "Buisness Development": "Business Development" } -# repository links -REPOSITORY_STUDY_ID_LINK_TEMPLATE = { - "NIDDK Central": "https://repository.niddk.nih.gov/studies//", - "NIDA Data Share": "https://datashare.nida.nih.gov/study/", - "NICHD DASH": "https://dash.nichd.nih.gov/study/", - "ICPSR": "https://www.icpsr.umich.edu/web/ICPSR/studies/", - "BioSystics-AP": "https://biosystics-ap.com/assays/assaystudy//", -} - - # Defines field that we don't want to include in the filters OMITTED_VALUES_MAPPING = { "study_metadata.human_subject_applicability.gender_applicability": "Not applicable" @@ -124,31 +114,6 @@ def get_client_token(client_id: str, client_secret: str): return token -def get_related_studies(serial_num, hostname): - related_study_result = [] - - if serial_num: - mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000") - if mds.status_code == 200: - related_study_metadata = mds.json() - - for ( - related_study_metadata_key, - related_study_metadata_value, - ) in related_study_metadata.items(): - title = ( - related_study_metadata_value.get( - "gen3_discovery", {} - ) - .get("study_metadata", {}) - .get("minimal_info", {}) - .get("study_name", "") - ) - link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/" - related_study_result.append({"title": title, "link": link}) - return related_study_result - - parser = argparse.ArgumentParser() parser.add_argument("--directory", help="CEDAR Directory ID for registering ") @@ -249,67 +214,6 @@ def get_related_studies(serial_num, hostname): mds_res["gen3_discovery"]["study_metadata"].update(cedar_record) mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites - # setup citations - doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "") - mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation - - - # setup repository_study_link - data_repositories = ( - mds_res.get("study_metadata", {}) - .get("metadata_location", {}) - .get("data_repositories", []) - ) - repository_citation = "Users must also include a citation to the data as specified by the local repository." - repository_citation_additional_text = ' The link to the study page at the local repository can be found in the "Data" tab.' - for repository in data_repositories: - if ( - repository["repository_name"] - and repository["repository_name"] - in REPOSITORY_STUDY_ID_LINK_TEMPLATE - and repository["repository_study_ID"] - ): - repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[ - repository["repository_name"] - ].replace("", repository["repository_study_ID"]) - repository.update({"repository_study_link": repository_study_link}) - if repository_citation_additional_text not in repository_citation: - repository_citation += repository_citation_additional_text - if len(data_repositories): - data_repositories[0] = { - **data_repositories[0], - "repository_citation": repository_citation, - } - mds_res["gen3_discovery"]["study_metadata"][ - "metadata_location" - ]["data_repositories"] = data_repositories - - - - # set up related studies - serial_num = None - try: - serial_num = ( - mds_res - .get("nih_reporter", {}) - .get("project_num_split", {}) - .get("serial_num", None) - ) - except Exception: - print(f"Unable to get serial number for study") - - if serial_num == None: - print(f"Unable to get serial number for study") - - related_study_result = get_related_studies(serial_num, hostname) - existing_related_study_result = mds_res.get("related_studies", []) - for related_study in related_study_result: - if related_study not in existing_related_study_result: - existing_related_study_result.append(copy.deepcopy(related_study)) - mds_res["gen3_discovery"][ - "related_studies" - ] = copy.deepcopy(existing_related_study_result) - # merge data from cedar that is not study level metadata into a level higher deleted_keys = [] for key, value in mds_res["gen3_discovery"]["study_metadata"].items(): From 6ee728ad3e375b1a0e521ae256ef13ca29529067 Mon Sep 17 00:00:00 2001 From: Alexander VanTol Date: Mon, 4 Mar 2024 11:04:31 -0600 Subject: [PATCH 074/114] Automation for Gen3 Discovery AI Service (#2396) * feat(gen3-openai): rough initial testing, no automation for rolling * feat(gen3-discovery-ai): initial deployment * fix(ai): fix setup jq escaping * fix(ai): fix file name * fix(ai): fix deployment configmap * fix(ai): fix configmap * fix(ai): env path * feat(image): use from manifest * chore(ai): better naming * fix(ai): fix mount path for cfg * fix(ai): first attempt to fix issue of needing write volume for chromadb persistance * fix(ai): k8s deploy command * fix(ai): fix duplicate name * chore(ai): don't sent telemetry data * chore(ai): more logging in init * chore(logs): more * fix(ai): mv instead of cp * fix(ai): back to cp, can't mv b/c of readonly * feat(ai): use s3 and service account + role to handle persisted vectorstore data instead of configmap * fix(ai): fix setup * fix(ai): fix setup * fix(ai): fix automation * fix(ai): automation * fix(ai): fix logic for setup * fix(ai): mount storage config and don't use gen3/jq since they're not available * fix(ai): fix wrong path * fix(ai): quotes * fix(ai): quoting * fix(ai): use awshelper for access to aws commands * fix(ai): move files to correct location * fix(ai): only get folder * fix(ai): fix sync * fix(ai): clear folder before syncing * fix(ai): update bucket contents every roll for updates * feat(ai): support TSV loading from manifest config * fix(ai): fix init so aws syncing is done with awshelper image and loading into vectorstore is with service image * fix(ai): fix loading * fix(ai): fix loading * fix(ai): sync all files * feat(ai): add google secret loading and mounting * fix(ai): mount to container, not inits * fix(mount): don't create another dir * fix(mount): don't create another dir * fix(mounts): fix paths * fix(mounts): mount all secrets * fix(secrets): allow .env file to be a secret * fix(secrets): revert failed attempt to support .env * chore(ai): cd to dir with pyproject.toml * chore(ai): try to fix issue with pyproject.toml * fix(ai): actually we need to poetry run * chore(ai): debug lines * chore(ai): debug lines * chore(ai): debug lines * chore(ai): debug lines * chore(ai): debug lines * fix(mount): don't overwrite whole dir * fix(ai): mounts * chore(ai): remove debug lines * fix(ai): remove debug * chore(debug): debug line * chore(debug): remove debug line * feat(ai): add to roll all, fix port in service yaml * fix(ai): fix nginx conf file name * fix(nginx): fix routing for AI service to add trailing slash after "ai" * Update web_whitelist * Update kube-setup-gen3-discovery-ai.sh * Update README.md * Update gen3-discovery-ai-deploy.yaml * Update gen3-discovery-ai-deploy.yaml * Update gen3-discovery-ai-service.yaml * Update kube-setup-gen3-discovery-ai.sh * feat(discovery): update to data load commands and strategy to support markdown --- gen3/bin/kube-roll-all.sh | 6 + gen3/bin/kube-setup-gen3-discovery-ai.sh | 154 +++++++++++++++ kube/services/gen3-discovery-ai/README.md | 42 ++++ .../gen3-discovery-ai-deploy.yaml | 181 ++++++++++++++++++ .../gen3-discovery-ai-service.yaml | 21 ++ .../gen3-discovery-ai-service.conf | 12 ++ 6 files changed, 416 insertions(+) create mode 100644 gen3/bin/kube-setup-gen3-discovery-ai.sh create mode 100644 kube/services/gen3-discovery-ai/README.md create mode 100644 kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml create mode 100644 kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml create mode 100644 kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf diff --git a/gen3/bin/kube-roll-all.sh b/gen3/bin/kube-roll-all.sh index 6a67f2bdd2..1dca87c68c 100644 --- a/gen3/bin/kube-roll-all.sh +++ b/gen3/bin/kube-roll-all.sh @@ -243,6 +243,12 @@ else gen3_log_info "not deploying dicom-viewer - no manifest entry for '.versions[\"dicom-viewer\"]'" fi +if g3k_manifest_lookup '.versions["gen3-discovery-ai"]' 2> /dev/null; then + gen3 kube-setup-gen3-discovery-ai & +else + gen3_log_info "not deploying gen3-discovery-ai - no manifest entry for '.versions[\"gen3-discovery-ai\"]'" +fi + if g3k_manifest_lookup '.versions["ohdsi-atlas"]' && g3k_manifest_lookup '.versions["ohdsi-webapi"]' 2> /dev/null; then gen3 kube-setup-ohdsi & else diff --git a/gen3/bin/kube-setup-gen3-discovery-ai.sh b/gen3/bin/kube-setup-gen3-discovery-ai.sh new file mode 100644 index 0000000000..44a472a74c --- /dev/null +++ b/gen3/bin/kube-setup-gen3-discovery-ai.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# +# Deploy the gen3-discovery-ai service +# + +source "${GEN3_HOME}/gen3/lib/utils.sh" +gen3_load "gen3/gen3setup" + +# NOTE: no db for this service yet, but we'll likely need it in the future +setup_database() { + gen3_log_info "setting up gen3-discovery-ai service ..." + + if g3kubectl describe secret gen3-discovery-ai-g3auto > /dev/null 2>&1; then + gen3_log_info "gen3-discovery-ai-g3auto secret already configured" + return 0 + fi + if [[ -n "$JENKINS_HOME" || ! -f "$(gen3_secrets_folder)/creds.json" ]]; then + gen3_log_err "skipping db setup in non-adminvm environment" + return 0 + fi + # Setup .env file that gen3-discovery-ai service consumes + if [[ ! -f "$secretsFolder/gen3-discovery-ai.env" || ! -f "$secretsFolder/base64Authz.txt" ]]; then + local secretsFolder="$(gen3_secrets_folder)/g3auto/gen3-discovery-ai" + + if [[ ! -f "$secretsFolder/dbcreds.json" ]]; then + if ! gen3 db setup gen3-discovery-ai; then + gen3_log_err "Failed setting up database for gen3-discovery-ai service" + return 1 + fi + fi + if [[ ! -f "$secretsFolder/dbcreds.json" ]]; then + gen3_log_err "dbcreds not present in Gen3Secrets/" + return 1 + fi + + # go ahead and rotate the password whenever we regen this file + local password="$(gen3 random)" + cat - > "$secretsFolder/gen3-discovery-ai.env" < "$secretsFolder/base64Authz.txt" + fi + gen3 secrets sync 'setup gen3-discovery-ai-g3auto secrets' +} + +if ! g3k_manifest_lookup '.versions."gen3-discovery-ai"' 2> /dev/null; then + gen3_log_info "kube-setup-gen3-discovery-ai exiting - gen3-discovery-ai service not in manifest" + exit 0 +fi + +# There's no db for this service *yet* +# +# if ! setup_database; then +# gen3_log_err "kube-setup-gen3-discovery-ai bailing out - database failed setup" +# exit 1 +# fi + +setup_storage() { + local saName="gen3-discovery-ai-sa" + g3kubectl create sa "$saName" > /dev/null 2>&1 || true + + local secret + local secretsFolder="$(gen3_secrets_folder)/g3auto/gen3-discovery-ai" + + secret="$(g3kubectl get secret gen3-discovery-ai-g3auto -o json 2> /dev/null)" + local hasStorageCfg + hasStorageCfg=$(jq -r '.data | has("storage_config.json")' <<< "$secret") + + if [ "$hasStorageCfg" = "false" ]; then + gen3_log_info "setting up storage for gen3-discovery-ai service" + # + # gen3-discovery-ai-g3auto secret still does not exist + # we need to setup an S3 bucket and IAM creds + # let's avoid creating multiple buckets for different + # deployments to the same k8s cluster (dev, etc) + # + local bucketName + local accountNumber + local environment + + if ! accountNumber="$(aws sts get-caller-identity --output text --query 'Account')"; then + gen3_log_err "could not determine account numer" + return 1 + fi + + gen3_log_info "accountNumber: ${accountNumber}" + + if ! environment="$(g3kubectl get configmap manifest-global -o json | jq -r .data.environment)"; then + gen3_log_err "could not determine environment from manifest-global - bailing out of gen3-discovery-ai setup" + return 1 + fi + + gen3_log_info "environment: ${environment}" + + # try to come up with a unique but composable bucket name + bucketName="gen3-discovery-ai-${accountNumber}-${environment//_/-}" + + gen3_log_info "bucketName: ${bucketName}" + + if aws s3 ls --page-size 1 "s3://${bucketName}" > /dev/null 2>&1; then + gen3_log_info "${bucketName} s3 bucket already exists - probably in use by another namespace - copy the creds from there to $(gen3_secrets_folder)/g3auto/gen3-discovery-ai" + # continue on ... + elif ! gen3 s3 create "${bucketName}"; then + gen3_log_err "maybe failed to create bucket ${bucketName}, but maybe not, because the terraform script is flaky" + fi + + local hostname + hostname="$(gen3 api hostname)" + jq -r -n --arg bucket "${bucketName}" --arg hostname "${hostname}" '.bucket=$bucket | .prefix=$hostname' > "${secretsFolder}/storage_config.json" + gen3 secrets sync 'setup gen3-discovery-ai credentials' + + local roleName + roleName="$(gen3 api safe-name gen3-discovery-ai)" || return 1 + + if ! gen3 awsrole info "$roleName" > /dev/null; then # setup role + bucketName="$( (gen3 secrets decode 'gen3-discovery-ai-g3auto' 'storage_config.json' || echo ERROR) | jq -r .bucket)" || return 1 + gen3 awsrole create "$roleName" "$saName" || return 1 + gen3 s3 attach-bucket-policy "$bucketName" --read-write --role-name "${roleName}" + # try to give the gitops role read/write permissions on the bucket + local gitopsRoleName + gitopsRoleName="$(gen3 api safe-name gitops)" + gen3 s3 attach-bucket-policy "$bucketName" --read-write --role-name "${gitopsRoleName}" + fi + fi + + return 0 +} + +if ! setup_storage; then + gen3_log_err "kube-setup-gen3-discovery-ai bailing out - storage failed setup" + exit 1 +fi + +gen3_log_info "Setup complete, syncing configuration to bucket" + +bucketName="$( (gen3 secrets decode 'gen3-discovery-ai-g3auto' 'storage_config.json' || echo ERROR) | jq -r .bucket)" || exit 1 +aws s3 sync "$(dirname $(g3k_manifest_path))/gen3-discovery-ai/knowledge" "s3://$bucketName" --delete + +gen3 roll gen3-discovery-ai +g3kubectl apply -f "${GEN3_HOME}/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml" + +if [[ -z "$GEN3_ROLL_ALL" ]]; then + gen3 kube-setup-networkpolicy + gen3 kube-setup-revproxy +fi + +gen3_log_info "The gen3-discovery-ai service has been deployed onto the kubernetes cluster" +gen3_log_info "test with: curl https://commons-host/ai" diff --git a/kube/services/gen3-discovery-ai/README.md b/kube/services/gen3-discovery-ai/README.md new file mode 100644 index 0000000000..4c20678e06 --- /dev/null +++ b/kube/services/gen3-discovery-ai/README.md @@ -0,0 +1,42 @@ +# Gen3 Discovery AI Configuration + +Expects data in a `gen3-discovery-ai` folder relative to +where the `manifest.json` is. + +Basic setup: + +`{{dir where manifest.json is}}/gen3-discovery-ai/knowledge/` + +- `tsvs` folder + - tsvs with topic_name at beginning of file +- `markdown` folder + - {{topic_name_1}} + - markdown file(s) + - {{topic_name_2}} + - markdown file(s) + +The `kube-setup-gen3-discovery-ai` script syncs the above `/knowledge` folder to +an S3 bucket. The service configuration then pulls from the S3 bucket and runs load commands +to get the data into chromadb. + +> Note: See the `gen3-discovery-ai` service repo docs and README for more details on data load capabilities. + +Check the `gen3-discovery-ai-deploy.yaml` for what commands are being run in the automation. + +Expects secrets setup in `g3auto/gen3-discovery-ai` folder + - `credentials.json`: Google service account key if using a topic with Google Vertex AI + - `env`: .env file contents for service configuration (see service repo for a default one) + +## Populating Disk for In-Memory Vectordb Chromadb + +In order to setup pre-configured topics, we need to load a bunch of data +into Chromadb (which is an in-mem vectordb with an option to persist to disk). + +To load topics consistently, we setup an S3 bucket to house the persisted +data for the vectordb. + +### Getting data from S3 in mem + +We specify a path for Chromadb to use for persisted data and when it sees +data there, it loads it in. So the deployment automation: 1. aws syncs the bucket +and then 2. calls a script to load the files into the in-mem vectorstore from there. diff --git a/kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml b/kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml new file mode 100644 index 0000000000..dcfe03248a --- /dev/null +++ b/kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml @@ -0,0 +1,181 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gen3-discovery-ai-deployment +spec: + selector: + # Only select pods based on the 'app' label + matchLabels: + app: gen3-discovery-ai + release: production + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: gen3-discovery-ai + release: production + GEN3_DATE_LABEL + spec: + serviceAccountName: gen3-discovery-ai-sa + volumes: + - name: gen3-discovery-ai-g3auto-volume + secret: + secretName: gen3-discovery-ai-g3auto + - name: gen3-discovery-ai-knowledge-library-volume + emptyDir: {} + initContainers: + # chromadb's persisted disk support requires the ability to write. We don't technically need this ability + # since we're populating the entirety of the database from configured files (no live updates). + # + # Solution: utilize emptyDir as a writable space. + # + # Procedure: in init containers, copy files from s3 to writable + # temporary space in emptyDir, use files from writable space + # to load into knowledge libary, move final knowledge library + # files into top-level emptyDir and make available in final container + - name: gen3-discovery-ai-aws-init + GEN3_AWSHELPER_IMAGE|-image: quay.io/cdis/awshelper:master-| + imagePullPolicy: Always + ports: + - containerPort: 8080 + env: + - name: GEN3_DEBUG + GEN3_DEBUG_FLAG|-value: "False"-| + volumeMounts: + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/.env + subPath: env + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/credentials.json + subPath: credentials.json + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/storage_config.json + subPath: storage_config.json + - name: gen3-discovery-ai-knowledge-library-volume + mountPath: /gen3discoveryai/knowledge + imagePullPolicy: Always + resources: + requests: + cpu: 1 + limits: + cpu: 2 + memory: 512Mi + command: ["/bin/bash"] + args: + - "-c" + - | + bucketName=$(grep -o "\"bucket\": *\"[^\"]*\"" /gen3discoveryai/storage_config.json | awk -F'"' '{print $4}') + echo BUCKET: "$bucketName" + echo + echo BEFORE /gen3discoveryai/knowledge + ls -Ra /gen3discoveryai/knowledge + echo + echo syncing from s3 + aws s3 sync "s3://${bucketName}" "/gen3discoveryai/knowledge/tmp" + echo + echo AFTER /gen3discoveryai/knowledge + ls -Ra /gen3discoveryai/knowledge + - name: gen3-discovery-ai-knowledge-init + GEN3_GEN3-DISCOVERY-AI_IMAGE + imagePullPolicy: Always + ports: + - containerPort: 8080 + env: + - name: GEN3_DEBUG + GEN3_DEBUG_FLAG|-value: "False"-| + - name: ANONYMIZED_TELEMETRY + value: "False" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /gen3discoveryai/credentials.json + volumeMounts: + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/.env + subPath: env + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/credentials.json + subPath: credentials.json + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/storage_config.json + subPath: storage_config.json + - name: gen3-discovery-ai-knowledge-library-volume + mountPath: /gen3discoveryai/knowledge + imagePullPolicy: Always + resources: + requests: + cpu: 1 + limits: + cpu: 2 + memory: 512Mi + command: ["/bin/bash"] + args: + - "-c" + - | + echo + echo BEFORE /gen3discoveryai/knowledge + ls -Ra /gen3discoveryai/knowledge + echo running load_into_knowledge_store.py + poetry run python /gen3discoveryai/bin/load_into_knowledge_store.py tsvs /gen3discoveryai/knowledge/tmp/tsvs + + if [ -d "/gen3discoveryai/knowledge/tmp/markdown" ]; then + for dir in "/gen3discoveryai/knowledge/tmp/markdown"/*; do + if [ -d "$dir" ]; then + dir_name=$(basename "$dir") + + echo "Processing directory: $dir_name. Full path: $dir" + poetry run python /gen3discoveryai/bin/load_into_knowledge_store.py markdown --topic $dir_name $dir + fi + done + else + echo "Not syncing markdown, directory not found: /gen3discoveryai/knowledge/tmp/markdown" + fi + + rm -r /gen3discoveryai/knowledge/tmp/ + echo + echo AFTER /gen3discoveryai/knowledge + ls -Ra /gen3discoveryai/knowledge + containers: + - name: gen3-discovery-ai + GEN3_GEN3-DISCOVERY-AI_IMAGE + imagePullPolicy: Always + ports: + - containerPort: 8080 + env: + - name: GEN3_DEBUG + GEN3_DEBUG_FLAG|-value: "False"-| + - name: ANONYMIZED_TELEMETRY + value: "False" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /gen3discoveryai/credentials.json + volumeMounts: + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/.env + subPath: env + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/credentials.json + subPath: credentials.json + - name: gen3-discovery-ai-g3auto-volume + readOnly: true + mountPath: /gen3discoveryai/storage_config.json + subPath: storage_config.json + - name: gen3-discovery-ai-knowledge-library-volume + mountPath: /gen3discoveryai/knowledge + imagePullPolicy: Always + resources: + requests: + cpu: 1 + limits: + cpu: 2 + # NOTE: If the configured data for the knowledge library (vector database) is large, you may need to bump this + memory: 512Mi diff --git a/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml b/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml new file mode 100644 index 0000000000..b4734c3b8a --- /dev/null +++ b/kube/services/gen3-discovery-ai/gen3-discovery-ai-service.yaml @@ -0,0 +1,21 @@ +kind: Service +apiVersion: v1 +metadata: + name: gen3-discovery-ai-service +spec: + selector: + app: gen3-discovery-ai + release: production + ports: + - protocol: TCP + port: 80 + targetPort: 8089 + name: http + nodePort: null + - protocol: TCP + port: 443 + targetPort: 443 + name: https + nodePort: null + type: ClusterIP + diff --git a/kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf b/kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf new file mode 100644 index 0000000000..42e9a3758b --- /dev/null +++ b/kube/services/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf @@ -0,0 +1,12 @@ + location /ai { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "gen3-discovery-ai-service"; + set $upstream http://gen3-discovery-ai-service$des_domain; + rewrite ^/ai/(.*) /$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/ai/; + client_max_body_size 0; + } From 077e475463098416001ebd3ced41ec06d1ee631b Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:21:16 -0500 Subject: [PATCH 075/114] Raising requests for Jenkins pods (#2495) --- .secrets.baseline | 2 +- Jenkinsfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index b7e06622d6..fbed122fd0 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-02-23T20:30:41Z" + "generated_at": "2024-02-23T20:30:41Z", "plugins_used": [ { "name": "AWSKeyDetector" diff --git a/Jenkinsfile b/Jenkinsfile index 4e3470ded6..9c70a2e378 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -99,8 +99,8 @@ spec: resources: requests: cpu: 0.2 - memory: 200Mi - ephemeral-storage: 200Mi + memory: 400Mi + ephemeral-storage: 1Gi env: - name: AWS_DEFAULT_REGION value: us-east-1 From 62894388c1ac907587916462d641788c87226e4a Mon Sep 17 00:00:00 2001 From: Michael Lukowski Date: Thu, 7 Mar 2024 10:42:36 -0600 Subject: [PATCH 076/114] fix cedar ingestion problems (#2494) * fix cedar ingestion problems * add default falue to get statement --------- Co-authored-by: Mingfei Shao <2475897+mfshao@users.noreply.github.com> --- .secrets.baseline | 360 +++++------------- .../healdata/heal-cedar-data-ingest.py | 101 +++++ 2 files changed, 187 insertions(+), 274 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index fbed122fd0..200b69841f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,9 +1,9 @@ { "exclude": { - "files": "^.secrets.baseline$", + "files": null, "lines": null }, - "generated_at": "2024-02-23T20:30:41Z", + "generated_at": "2024-03-04T21:42:56Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -61,14 +61,12 @@ "Chef/repo/data_bags/README.md": [ { "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366", - "is_secret": false, "is_verified": false, "line_number": 45, "type": "Secret Keyword" }, { "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee", - "is_secret": false, "is_verified": false, "line_number": 51, "type": "Secret Keyword" @@ -77,25 +75,22 @@ "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, - "line_number": 121, + "line_number": 124, "type": "Secret Keyword" } ], "Docker/jenkins/Jenkins-Worker/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, - "line_number": 143, + "line_number": 139, "type": "Secret Keyword" } ], "Docker/jenkins/Jenkins/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, "line_number": 107, "type": "Secret Keyword" @@ -104,7 +99,6 @@ "Docker/jenkins/Jenkins2/Dockerfile": [ { "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", - "is_secret": false, "is_verified": false, "line_number": 108, "type": "Secret Keyword" @@ -113,7 +107,6 @@ "Docker/sidecar/service.key": [ { "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", - "is_secret": false, "is_verified": false, "line_number": 1, "type": "Private Key" @@ -122,7 +115,6 @@ "Jenkins/Stacks/Jenkins/jenkins.env.sample": [ { "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Secret Keyword" @@ -131,7 +123,6 @@ "ansible/roles/awslogs/defaults/main.yaml": [ { "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Basic Auth Credentials" @@ -140,14 +131,12 @@ "ansible/roles/slurm/README.md": [ { "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42", - "is_secret": false, "is_verified": false, "line_number": 86, "type": "Base64 High Entropy String" }, { "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476", - "is_secret": false, "is_verified": false, "line_number": 86, "type": "Secret Keyword" @@ -156,7 +145,6 @@ "apis_configs/config_helper.py": [ { "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", - "is_secret": false, "is_verified": false, "line_number": 66, "type": "Basic Auth Credentials" @@ -165,7 +153,6 @@ "apis_configs/fence_credentials.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 23, "type": "Secret Keyword" @@ -174,21 +161,18 @@ "apis_configs/fence_settings.py": [ { "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Basic Auth Credentials" }, { "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3", - "is_secret": false, "is_verified": false, "line_number": 58, "type": "Secret Keyword" }, { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", - "is_secret": false, "is_verified": false, "line_number": 80, "type": "Basic Auth Credentials" @@ -197,7 +181,6 @@ "apis_configs/indexd_settings.py": [ { "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc", - "is_secret": false, "is_verified": false, "line_number": 59, "type": "Basic Auth Credentials" @@ -206,7 +189,6 @@ "apis_configs/peregrine_settings.py": [ { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", - "is_secret": false, "is_verified": false, "line_number": 46, "type": "Basic Auth Credentials" @@ -215,7 +197,6 @@ "apis_configs/sheepdog_settings.py": [ { "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", - "is_secret": false, "is_verified": false, "line_number": 46, "type": "Basic Auth Credentials" @@ -224,7 +205,6 @@ "doc/Gen3-data-upload.md": [ { "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" @@ -233,7 +213,6 @@ "doc/api.md": [ { "hashed_secret": "625de83a7517422051911680cc803921ff99db90", - "is_secret": false, "is_verified": false, "line_number": 47, "type": "Hex High Entropy String" @@ -242,28 +221,24 @@ "doc/gen3OnK8s.md": [ { "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069", - "is_secret": false, "is_verified": false, "line_number": 113, "type": "Secret Keyword" }, { "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2", - "is_secret": false, "is_verified": false, "line_number": 143, "type": "Secret Keyword" }, { "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48", - "is_secret": false, "is_verified": false, "line_number": 170, "type": "Secret Keyword" }, { "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce", - "is_secret": false, "is_verified": false, "line_number": 189, "type": "Secret Keyword" @@ -272,7 +247,6 @@ "doc/kube-setup-data-ingestion-job.md": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Secret Keyword" @@ -281,7 +255,6 @@ "doc/logs.md": [ { "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Secret Keyword" @@ -290,7 +263,6 @@ "doc/slurm_cluster.md": [ { "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", - "is_secret": false, "is_verified": false, "line_number": 184, "type": "Secret Keyword" @@ -299,14 +271,12 @@ "files/dashboard/usage-reports/package-lock.json": [ { "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448", - "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" }, { "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b", - "is_secret": false, "is_verified": false, "line_number": 15, "type": "Base64 High Entropy String" @@ -315,14 +285,12 @@ "gen3/bin/api.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 407, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", - "is_secret": false, "is_verified": false, "line_number": 477, "type": "Secret Keyword" @@ -331,7 +299,6 @@ "gen3/bin/kube-dev-namespace.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 135, "type": "Secret Keyword" @@ -340,7 +307,6 @@ "gen3/bin/kube-setup-argo.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 206, "type": "Secret Keyword" @@ -349,7 +315,6 @@ "gen3/bin/kube-setup-aurora-monitoring.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 59, "type": "Secret Keyword" @@ -358,7 +323,6 @@ "gen3/bin/kube-setup-certs.sh": [ { "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99", - "is_secret": false, "is_verified": false, "line_number": 50, "type": "Secret Keyword" @@ -367,14 +331,12 @@ "gen3/bin/kube-setup-dashboard.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 40, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", - "is_secret": false, "is_verified": false, "line_number": 41, "type": "Secret Keyword" @@ -383,14 +345,12 @@ "gen3/bin/kube-setup-data-ingestion-job.sh": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 37, "type": "Secret Keyword" }, { "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c", - "is_secret": false, "is_verified": false, "line_number": 102, "type": "Secret Keyword" @@ -399,7 +359,6 @@ "gen3/bin/kube-setup-dicom-server.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 43, "type": "Secret Keyword" @@ -408,23 +367,48 @@ "gen3/bin/kube-setup-dicom.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" } ], + "gen3/bin/kube-setup-gen3-discovery-ai.sh": [ + { + "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "is_verified": false, + "line_number": 37, + "type": "Secret Keyword" + }, + { + "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "is_verified": false, + "line_number": 71, + "type": "Secret Keyword" + } + ], "gen3/bin/kube-setup-jenkins.sh": [ { "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, + "is_verified": false, + "line_number": 22, + "type": "Secret Keyword" + } + ], + "gen3/bin/kube-setup-jenkins2.sh": [ + { + "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", + "is_verified": false, + "line_number": 18, + "type": "Secret Keyword" + }, + { + "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", "is_verified": false, "line_number": 22, "type": "Secret Keyword" @@ -433,7 +417,6 @@ "gen3/bin/kube-setup-metadata.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 35, "type": "Secret Keyword" @@ -442,21 +425,18 @@ "gen3/bin/kube-setup-revproxy.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 38, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 55, "type": "Secret Keyword" }, { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 57, "type": "Secret Keyword" @@ -465,21 +445,18 @@ "gen3/bin/kube-setup-secrets.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 79, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 82, "type": "Secret Keyword" }, { "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8", - "is_secret": false, "is_verified": false, "line_number": 95, "type": "Secret Keyword" @@ -488,14 +465,12 @@ "gen3/bin/kube-setup-sftp.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 36, "type": "Secret Keyword" }, { "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4", - "is_secret": false, "is_verified": false, "line_number": 51, "type": "Secret Keyword" @@ -504,7 +479,6 @@ "gen3/bin/kube-setup-sheepdog.sh": [ { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 33, "type": "Secret Keyword" @@ -513,28 +487,24 @@ "gen3/bin/kube-setup-sower-jobs.sh": [ { "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", - "is_secret": false, "is_verified": false, "line_number": 25, "type": "Secret Keyword" }, { "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 120, "type": "Secret Keyword" }, { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 122, "type": "Secret Keyword" @@ -543,21 +513,18 @@ "gen3/bin/kube-setup-ssjdispatcher.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 117, "type": "Secret Keyword" }, { "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93", - "is_secret": false, "is_verified": false, "line_number": 184, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 197, "type": "Secret Keyword" @@ -566,14 +533,12 @@ "gen3/lib/aws.sh": [ { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", - "is_secret": false, "is_verified": false, "line_number": 640, "type": "Secret Keyword" }, { "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8", - "is_secret": false, "is_verified": false, "line_number": 660, "type": "Secret Keyword" @@ -582,14 +547,12 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [ { "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", - "is_secret": false, "is_verified": false, "line_number": 33, "type": "Basic Auth Credentials" }, { "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db", - "is_secret": false, "is_verified": false, "line_number": 286, "type": "Secret Keyword" @@ -598,7 +561,6 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" @@ -607,7 +569,6 @@ "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Secret Keyword" @@ -616,7 +577,6 @@ "gen3/lib/logs/utils.sh": [ { "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5", - "is_secret": false, "is_verified": false, "line_number": 3, "type": "Secret Keyword" @@ -625,7 +585,6 @@ "gen3/lib/manifestDefaults/hatchery/hatchery.json": [ { "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039", - "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -634,14 +593,12 @@ "gen3/lib/onprem.sh": [ { "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3", - "is_secret": false, "is_verified": false, "line_number": 68, "type": "Secret Keyword" }, { "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70", - "is_secret": false, "is_verified": false, "line_number": 84, "type": "Secret Keyword" @@ -650,14 +607,12 @@ "gen3/lib/secrets/rotate-postgres.sh": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 162, "type": "Secret Keyword" }, { "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", - "is_secret": false, "is_verified": false, "line_number": 250, "type": "Secret Keyword" @@ -666,49 +621,42 @@ "gen3/lib/testData/etlconvert/expected2.yaml": [ { "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474", - "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" }, { "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55", - "is_secret": false, "is_verified": false, "line_number": 13, "type": "Base64 High Entropy String" }, { "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26", - "is_secret": false, "is_verified": false, "line_number": 16, "type": "Base64 High Entropy String" }, { "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424", - "is_secret": false, "is_verified": false, "line_number": 33, "type": "Base64 High Entropy String" }, { "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba", - "is_secret": false, "is_verified": false, "line_number": 35, "type": "Base64 High Entropy String" }, { "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22", - "is_secret": false, "is_verified": false, "line_number": 36, "type": "Base64 High Entropy String" @@ -717,7 +665,6 @@ "gen3/test/secretsTest.sh": [ { "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938", - "is_secret": false, "is_verified": false, "line_number": 25, "type": "Secret Keyword" @@ -726,28 +673,24 @@ "gen3/test/terraformTest.sh": [ { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", - "is_secret": false, "is_verified": false, "line_number": 156, "type": "Secret Keyword" }, { "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009", - "is_secret": false, "is_verified": false, "line_number": 172, "type": "Base64 High Entropy String" }, { "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef", - "is_secret": false, "is_verified": false, "line_number": 175, "type": "Base64 High Entropy String" }, { "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c", - "is_secret": false, "is_verified": false, "line_number": 175, "type": "Secret Keyword" @@ -756,21 +699,18 @@ "kube/services/argocd/values.yaml": [ { "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd", - "is_secret": false, "is_verified": false, "line_number": 360, "type": "Private Key" }, { "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61", - "is_secret": false, "is_verified": false, "line_number": 379, "type": "Secret Keyword" }, { "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2", - "is_secret": false, "is_verified": false, "line_number": 412, "type": "Secret Keyword" @@ -779,7 +719,6 @@ "kube/services/datadog/values.yaml": [ { "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b", - "is_secret": false, "is_verified": false, "line_number": 23, "type": "Secret Keyword" @@ -788,401 +727,362 @@ "kube/services/fenceshib/fenceshib-configmap.yaml": [ { "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c", - "is_secret": false, "is_verified": false, "line_number": 375, "type": "Base64 High Entropy String" }, { "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f", - "is_secret": false, "is_verified": false, "line_number": 376, "type": "Base64 High Entropy String" }, { "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b", - "is_secret": false, "is_verified": false, "line_number": 377, "type": "Base64 High Entropy String" }, { "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5", - "is_secret": false, "is_verified": false, "line_number": 378, "type": "Base64 High Entropy String" }, { "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a", - "is_secret": false, "is_verified": false, "line_number": 379, "type": "Base64 High Entropy String" }, { "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9", - "is_secret": false, "is_verified": false, "line_number": 380, "type": "Base64 High Entropy String" }, { "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12", - "is_secret": false, "is_verified": false, "line_number": 381, "type": "Base64 High Entropy String" }, { "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7", - "is_secret": false, "is_verified": false, "line_number": 382, "type": "Base64 High Entropy String" }, { "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a", - "is_secret": false, "is_verified": false, "line_number": 383, "type": "Base64 High Entropy String" }, { "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3", - "is_secret": false, "is_verified": false, "line_number": 384, "type": "Base64 High Entropy String" }, { "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c", - "is_secret": false, "is_verified": false, "line_number": 385, "type": "Base64 High Entropy String" }, { "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52", - "is_secret": false, "is_verified": false, "line_number": 386, "type": "Base64 High Entropy String" }, { "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110", - "is_secret": false, "is_verified": false, "line_number": 387, "type": "Base64 High Entropy String" }, { "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7", - "is_secret": false, "is_verified": false, "line_number": 388, "type": "Base64 High Entropy String" }, { "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e", - "is_secret": false, "is_verified": false, "line_number": 389, "type": "Base64 High Entropy String" }, { "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1", - "is_secret": false, "is_verified": false, "line_number": 390, "type": "Base64 High Entropy String" }, { "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53", - "is_secret": false, "is_verified": false, "line_number": 391, "type": "Base64 High Entropy String" }, { "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a", - "is_secret": false, "is_verified": false, "line_number": 392, "type": "Base64 High Entropy String" }, { "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294", - "is_secret": false, "is_verified": false, "line_number": 393, "type": "Base64 High Entropy String" }, { "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41", - "is_secret": false, "is_verified": false, "line_number": 394, "type": "Base64 High Entropy String" }, { "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1", - "is_secret": false, "is_verified": false, "line_number": 395, "type": "Base64 High Entropy String" }, { "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb", - "is_secret": false, "is_verified": false, "line_number": 396, "type": "Base64 High Entropy String" }, { "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433", - "is_secret": false, "is_verified": false, "line_number": 397, "type": "Base64 High Entropy String" }, { "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a", - "is_secret": false, "is_verified": false, "line_number": 398, "type": "Base64 High Entropy String" }, { "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2", - "is_secret": false, "is_verified": false, "line_number": 399, "type": "Base64 High Entropy String" }, { - "hashed_secret": "7803ae08cdc22a5e0b025eff3c9ef0628eedc165", - "is_secret": false, + "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525", "is_verified": false, "line_number": 419, "type": "Base64 High Entropy String" }, { - "hashed_secret": "b8b61e87f5b58b0eeb597b2122ea0cea2ccab3d9", - "is_secret": false, + "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab", "is_verified": false, "line_number": 420, "type": "Base64 High Entropy String" }, { - "hashed_secret": "787745fc904c3bd7eddc3d1aab683a376c13890f", - "is_secret": false, + "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf", "is_verified": false, "line_number": 423, "type": "Base64 High Entropy String" }, { - "hashed_secret": "81361d672f238f505a6246ef9b655ee2f48d67e7", - "is_secret": false, + "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8", "is_verified": false, "line_number": 424, "type": "Base64 High Entropy String" }, { - "hashed_secret": "7c98bff76ac3f273d15ed9bc3dd5294d323ab577", - "is_secret": false, + "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2", "is_verified": false, "line_number": 425, "type": "Base64 High Entropy String" }, { - "hashed_secret": "46038fc88daceed8dd46817ca45c72ae0270fdd4", - "is_secret": false, + "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa", "is_verified": false, "line_number": 426, "type": "Base64 High Entropy String" }, { - "hashed_secret": "acad0c57b4f5cbed1b4863ed06d02784180a9f92", - "is_secret": false, + "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b", "is_verified": false, "line_number": 427, "type": "Base64 High Entropy String" }, { - "hashed_secret": "1b57f49a6ee337c16ecd6aabfc0dff3b3821cd09", - "is_secret": false, + "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f", "is_verified": false, "line_number": 428, "type": "Base64 High Entropy String" }, { - "hashed_secret": "5b688158be36e8b3f265a462ed599dcf69290084", - "is_secret": false, + "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48", "is_verified": false, "line_number": 429, "type": "Base64 High Entropy String" }, { - "hashed_secret": "965996e12c8b50b3c325d96003e8984a4ece658a", - "is_secret": false, + "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f", "is_verified": false, "line_number": 430, "type": "Base64 High Entropy String" }, { - "hashed_secret": "584f0c58e764e948af1a35c9e60447aa0f84c6f5", - "is_secret": false, + "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e", "is_verified": false, "line_number": 431, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bcaf897786d060a675ee9d654a84ae8baf96e9d0", - "is_secret": false, + "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46", "is_verified": false, "line_number": 432, "type": "Base64 High Entropy String" }, { - "hashed_secret": "0c09277fa183e06d32065f9386a3b4190b445df3", - "is_secret": false, + "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8", "is_verified": false, "line_number": 433, "type": "Base64 High Entropy String" }, { - "hashed_secret": "5a51be06b305d6664e4afd25f21869b0f8b5039b", - "is_secret": false, + "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d", "is_verified": false, "line_number": 434, "type": "Base64 High Entropy String" }, { - "hashed_secret": "b38404f8853d734e3d03577b2c1084b4540c8708", - "is_secret": false, + "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15", "is_verified": false, "line_number": 435, "type": "Base64 High Entropy String" }, { - "hashed_secret": "126ccc602cffcb8292beb57137f7f6719e317b72", - "is_secret": false, + "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f", "is_verified": false, "line_number": 436, "type": "Base64 High Entropy String" }, { - "hashed_secret": "6681c1d7e1d327642a32cb8864ad51e4b8f981e5", - "is_secret": false, + "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811", "is_verified": false, "line_number": 437, "type": "Base64 High Entropy String" }, { - "hashed_secret": "7f7b1f316ece195e5f584fe2faf6f9edc6942c6f", - "is_secret": false, + "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1", + "is_verified": false, + "line_number": 438, + "type": "Base64 High Entropy String" + }, + { + "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8", "is_verified": false, "line_number": 439, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bb908c7bc655057f2edc42815c5dff82e9dea529", - "is_secret": false, + "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f", "is_verified": false, "line_number": 440, "type": "Base64 High Entropy String" }, { - "hashed_secret": "bc2a0d18e3dd142df7b34e95342d47bf8aadabcb", - "is_secret": false, + "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa", "is_verified": false, "line_number": 441, "type": "Base64 High Entropy String" }, { - "hashed_secret": "d60f0bcea109bb6edb6e45fd387f5f2c86e49e1a", - "is_secret": false, + "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870", "is_verified": false, "line_number": 442, "type": "Base64 High Entropy String" }, { - "hashed_secret": "e549dd40a741557cc1c4e377df0a141354e22688", - "is_secret": false, + "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383", "is_verified": false, "line_number": 443, "type": "Base64 High Entropy String" }, { - "hashed_secret": "2dd2486dae84cad50387c20bf687b6fbc6162b58", - "is_secret": false, + "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7", "is_verified": false, "line_number": 444, "type": "Base64 High Entropy String" }, { - "hashed_secret": "71622010fc7eb09d9273f59c548bde6a5da5dc0e", - "is_secret": false, + "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8", "is_verified": false, "line_number": 445, "type": "Base64 High Entropy String" }, { - "hashed_secret": "6f0115cf53bd49ec990c562ac6cbfc452c83cd46", - "is_secret": false, + "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106", "is_verified": false, "line_number": 446, "type": "Base64 High Entropy String" }, { - "hashed_secret": "70dddd534b2f9bb70871fefe0845b79c3b69363f", - "is_secret": false, + "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554", + "is_verified": false, + "line_number": 447, + "type": "Base64 High Entropy String" + }, + { + "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413", "is_verified": false, "line_number": 448, "type": "Base64 High Entropy String" }, { - "hashed_secret": "acf3536b0416aa99608b0be17e87655370ece829", - "is_secret": false, + "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1", "is_verified": false, - "line_number": 449, + "line_number": 451, "type": "Base64 High Entropy String" }, { - "hashed_secret": "1d13ee35c7279c1fae1c6474ed47611994273e41", - "is_secret": false, + "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5", "is_verified": false, - "line_number": 450, + "line_number": 452, "type": "Base64 High Entropy String" }, { - "hashed_secret": "d38cf89b25bd7378cdb4e00b4b59293001dd500b", - "is_secret": false, + "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963", "is_verified": false, - "line_number": 451, + "line_number": 453, "type": "Base64 High Entropy String" }, { - "hashed_secret": "1648f34ce2f1b563a8ed1c6d5d55b5e76a395903", - "is_secret": false, + "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591", "is_verified": false, - "line_number": 452, + "line_number": 454, "type": "Base64 High Entropy String" }, { - "hashed_secret": "9bf63f6f49fb01ff80959bc5a60c8688df92cc02", - "is_secret": false, + "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911", "is_verified": false, - "line_number": 453, + "line_number": 455, + "type": "Base64 High Entropy String" + }, + { + "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a", + "is_verified": false, + "line_number": 456, "type": "Base64 High Entropy String" } ], "kube/services/jobs/indexd-authz-job.yaml": [ { "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", - "is_secret": false, "is_verified": false, "line_number": 87, "type": "Basic Auth Credentials" @@ -1191,14 +1091,12 @@ "kube/services/monitoring/grafana-values.yaml": [ { "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385", - "is_secret": false, "is_verified": false, "line_number": 162, "type": "Secret Keyword" }, { "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4", - "is_secret": false, "is_verified": false, "line_number": 166, "type": "Secret Keyword" @@ -1207,7 +1105,6 @@ "kube/services/revproxy/helpers.js": [ { "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af", - "is_secret": false, "is_verified": false, "line_number": 10, "type": "Base64 High Entropy String" @@ -1216,7 +1113,6 @@ "kube/services/revproxy/helpersTest.js": [ { "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb", - "is_secret": false, "is_verified": false, "line_number": 22, "type": "JSON Web Token" @@ -1225,7 +1121,6 @@ "kube/services/superset/superset-deploy.yaml": [ { "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7", - "is_secret": false, "is_verified": false, "line_number": 38, "type": "Secret Keyword" @@ -1234,14 +1129,12 @@ "kube/services/superset/superset-redis.yaml": [ { "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f", - "is_secret": false, "is_verified": false, "line_number": 165, "type": "Secret Keyword" }, { "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9", - "is_secret": false, "is_verified": false, "line_number": 265, "type": "Secret Keyword" @@ -1250,35 +1143,30 @@ "kube/services/superset/values.yaml": [ { "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3", - "is_secret": false, "is_verified": false, "line_number": 54, "type": "Secret Keyword" }, { "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46", - "is_secret": false, "is_verified": false, "line_number": 86, "type": "Secret Keyword" }, { "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8", - "is_secret": false, "is_verified": false, "line_number": 212, "type": "Secret Keyword" }, { "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358", - "is_secret": false, "is_verified": false, "line_number": 396, "type": "Secret Keyword" }, { "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc", - "is_secret": false, "is_verified": false, "line_number": 503, "type": "Secret Keyword" @@ -1287,280 +1175,240 @@ "package-lock.json": [ { "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d", - "is_secret": false, "is_verified": false, "line_number": 1481, "type": "Base64 High Entropy String" }, { "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87", - "is_secret": false, "is_verified": false, "line_number": 1489, "type": "Base64 High Entropy String" }, { "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00", - "is_secret": false, "is_verified": false, "line_number": 1521, "type": "Base64 High Entropy String" }, { "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04", - "is_secret": false, "is_verified": false, "line_number": 1547, "type": "Base64 High Entropy String" }, { "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af", - "is_secret": false, "is_verified": false, "line_number": 1611, "type": "Base64 High Entropy String" }, { "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93", - "is_secret": false, "is_verified": false, "line_number": 1640, "type": "Base64 High Entropy String" }, { "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e", - "is_secret": false, "is_verified": false, "line_number": 1648, "type": "Base64 High Entropy String" }, { "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e", - "is_secret": false, "is_verified": false, "line_number": 1664, "type": "Base64 High Entropy String" }, { "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e", - "is_secret": false, "is_verified": false, "line_number": 1683, "type": "Base64 High Entropy String" }, { "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56", - "is_secret": false, "is_verified": false, "line_number": 1733, "type": "Base64 High Entropy String" }, { "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3", - "is_secret": false, "is_verified": false, "line_number": 1742, "type": "Base64 High Entropy String" }, { "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e", - "is_secret": false, "is_verified": false, "line_number": 1755, "type": "Base64 High Entropy String" }, { "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85", - "is_secret": false, "is_verified": false, "line_number": 1769, "type": "Base64 High Entropy String" }, { "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72", - "is_secret": false, "is_verified": false, "line_number": 1855, "type": "Base64 High Entropy String" }, { "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5", - "is_secret": false, "is_verified": false, "line_number": 1889, "type": "Base64 High Entropy String" }, { "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c", - "is_secret": false, "is_verified": false, "line_number": 1894, "type": "Base64 High Entropy String" }, { "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c", - "is_secret": false, "is_verified": false, "line_number": 1921, "type": "Base64 High Entropy String" }, { "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620", - "is_secret": false, "is_verified": false, "line_number": 1950, "type": "Base64 High Entropy String" }, { "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71", - "is_secret": false, "is_verified": false, "line_number": 1983, "type": "Base64 High Entropy String" }, { "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d", - "is_secret": false, "is_verified": false, "line_number": 2004, "type": "Base64 High Entropy String" }, { "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6", - "is_secret": false, "is_verified": false, "line_number": 2013, "type": "Base64 High Entropy String" }, { "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b", - "is_secret": false, "is_verified": false, "line_number": 2046, "type": "Base64 High Entropy String" }, { "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466", - "is_secret": false, "is_verified": false, "line_number": 2078, "type": "Base64 High Entropy String" }, { "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f", - "is_secret": false, "is_verified": false, "line_number": 2083, "type": "Base64 High Entropy String" }, { "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265", - "is_secret": false, "is_verified": false, "line_number": 2111, "type": "Base64 High Entropy String" }, { "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc", - "is_secret": false, "is_verified": false, "line_number": 2138, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759", - "is_secret": false, "is_verified": false, "line_number": 2143, "type": "Base64 High Entropy String" }, { "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523", - "is_secret": false, "is_verified": false, "line_number": 2150, "type": "Base64 High Entropy String" }, { "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430", - "is_secret": false, "is_verified": false, "line_number": 2188, "type": "Base64 High Entropy String" }, { "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8", - "is_secret": false, "is_verified": false, "line_number": 2291, "type": "Base64 High Entropy String" }, { "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f", - "is_secret": false, "is_verified": false, "line_number": 2303, "type": "Base64 High Entropy String" }, { "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672", - "is_secret": false, "is_verified": false, "line_number": 2308, "type": "Base64 High Entropy String" }, { "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539", - "is_secret": false, "is_verified": false, "line_number": 2317, "type": "Base64 High Entropy String" }, { "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0", - "is_secret": false, "is_verified": false, "line_number": 2327, "type": "Base64 High Entropy String" }, { "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671", - "is_secret": false, "is_verified": false, "line_number": 2332, "type": "Base64 High Entropy String" }, { "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97", - "is_secret": false, "is_verified": false, "line_number": 2340, "type": "Base64 High Entropy String" }, { "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd", - "is_secret": false, "is_verified": false, "line_number": 2349, "type": "Base64 High Entropy String" }, { "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14", - "is_secret": false, "is_verified": false, "line_number": 2357, "type": "Base64 High Entropy String" }, { "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471", - "is_secret": false, "is_verified": false, "line_number": 2369, "type": "Base64 High Entropy String" }, { "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136", - "is_secret": false, "is_verified": false, "line_number": 2387, "type": "Base64 High Entropy String" @@ -1569,7 +1417,6 @@ "tf_files/aws/cognito/README.md": [ { "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70", - "is_secret": false, "is_verified": false, "line_number": 106, "type": "Secret Keyword" @@ -1578,14 +1425,12 @@ "tf_files/aws/commons/README.md": [ { "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed", - "is_secret": false, "is_verified": false, "line_number": 60, "type": "Secret Keyword" }, { "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e", - "is_secret": false, "is_verified": false, "line_number": 78, "type": "Secret Keyword" @@ -1594,7 +1439,6 @@ "tf_files/aws/eks/sample.tfvars": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", - "is_secret": false, "is_verified": false, "line_number": 107, "type": "Hex High Entropy String" @@ -1603,7 +1447,6 @@ "tf_files/aws/eks/variables.tf": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", - "is_secret": false, "is_verified": false, "line_number": 133, "type": "Hex High Entropy String" @@ -1612,14 +1455,12 @@ "tf_files/aws/modules/common-logging/README.md": [ { "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", - "is_secret": false, "is_verified": false, "line_number": 57, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", - "is_secret": false, "is_verified": false, "line_number": 59, "type": "Hex High Entropy String" @@ -1628,28 +1469,24 @@ "tf_files/aws/modules/common-logging/lambda_function.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Hex High Entropy String" @@ -1658,21 +1495,18 @@ "tf_files/aws/modules/common-logging/testLambda.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" @@ -1681,7 +1515,6 @@ "tf_files/aws/modules/eks/variables.tf": [ { "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", - "is_secret": false, "is_verified": false, "line_number": 113, "type": "Hex High Entropy String" @@ -1690,14 +1523,12 @@ "tf_files/aws/modules/management-logs/README.md": [ { "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", - "is_secret": false, "is_verified": false, "line_number": 54, "type": "Base64 High Entropy String" }, { "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", - "is_secret": false, "is_verified": false, "line_number": 56, "type": "Hex High Entropy String" @@ -1706,28 +1537,24 @@ "tf_files/aws/modules/management-logs/lambda_function.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 18, "type": "Hex High Entropy String" }, { "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", - "is_secret": false, "is_verified": false, "line_number": 30, "type": "Hex High Entropy String" @@ -1736,42 +1563,36 @@ "tf_files/aws/modules/management-logs/testLambda.py": [ { "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Base64 High Entropy String" }, { "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", - "is_secret": false, "is_verified": false, "line_number": 5, "type": "Hex High Entropy String" }, { "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Base64 High Entropy String" }, { "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Hex High Entropy String" }, { "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4", - "is_secret": false, "is_verified": false, "line_number": 6, "type": "Hex High Entropy String" @@ -1780,7 +1601,6 @@ "tf_files/aws/slurm/README.md": [ { "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d", - "is_secret": false, "is_verified": false, "line_number": 83, "type": "Secret Keyword" @@ -1789,7 +1609,6 @@ "tf_files/azure/cloud.tf": [ { "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4", - "is_secret": false, "is_verified": false, "line_number": 424, "type": "Secret Keyword" @@ -1798,7 +1617,6 @@ "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", - "is_secret": false, "is_verified": false, "line_number": 231, "type": "Secret Keyword" @@ -1807,7 +1625,6 @@ "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", - "is_secret": false, "is_verified": false, "line_number": 231, "type": "Secret Keyword" @@ -1816,7 +1633,6 @@ "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [ { "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", - "is_secret": false, "is_verified": false, "line_number": 262, "type": "Secret Keyword" @@ -1825,21 +1641,18 @@ "tf_files/gcp/commons/sample.tfvars": [ { "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", - "is_secret": false, "is_verified": false, "line_number": 11, "type": "Secret Keyword" }, { "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", - "is_secret": false, "is_verified": false, "line_number": 26, "type": "Secret Keyword" }, { "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105", - "is_secret": false, "is_verified": false, "line_number": 37, "type": "Secret Keyword" @@ -1848,7 +1661,6 @@ "tf_files/shared/modules/k8s_configs/creds.tpl": [ { "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", - "is_secret": false, "is_verified": false, "line_number": 8, "type": "Secret Keyword" diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index 71575e3c56..c54f9d5aa0 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -1,4 +1,5 @@ import argparse +import copy import json import sys import requests @@ -40,6 +41,16 @@ "study_metadata.human_subject_applicability.gender_applicability": "Not applicable" } +# repository links +REPOSITORY_STUDY_ID_LINK_TEMPLATE = { + "NIDDK Central": "https://repository.niddk.nih.gov/studies//", + "NIDA Data Share": "https://datashare.nida.nih.gov/study/", + "NICHD DASH": "https://dash.nichd.nih.gov/study/", + "ICPSR": "https://www.icpsr.umich.edu/web/ICPSR/studies/", + "BioSystics-AP": "https://biosystics-ap.com/assays/assaystudy//", +} + + def is_valid_uuid(uuid_to_test, version=4): """ Check if uuid_to_test is a valid UUID. @@ -114,6 +125,31 @@ def get_client_token(client_id: str, client_secret: str): return token +def get_related_studies(serial_num, hostname): + related_study_result = [] + + if serial_num: + mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000") + if mds.status_code == 200: + related_study_metadata = mds.json() + + for ( + related_study_metadata_key, + related_study_metadata_value, + ) in related_study_metadata.items(): + title = ( + related_study_metadata_value.get( + "gen3_discovery", {} + ) + .get("study_metadata", {}) + .get("minimal_info", {}) + .get("study_name", "") + ) + link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/" + related_study_result.append({"title": title, "link": link}) + return related_study_result + + parser = argparse.ArgumentParser() parser.add_argument("--directory", help="CEDAR Directory ID for registering ") @@ -214,6 +250,71 @@ def get_client_token(client_id: str, client_secret: str): mds_res["gen3_discovery"]["study_metadata"].update(cedar_record) mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites + # setup citations + doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "") + mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation + + + # setup repository_study_link + data_repositories = ( + mds_res + .get("gen3_discovery", {}) + .get("study_metadata", {}) + .get("metadata_location", {}) + .get("data_repositories", []) + ) + repository_citation = "Users must also include a citation to the data as specified by the local repository." + repository_citation_additional_text = ' The link to the study page at the local repository can be found in the "Data" tab.' + for repository in data_repositories: + if ( + repository["repository_name"] + and repository["repository_name"] + in REPOSITORY_STUDY_ID_LINK_TEMPLATE + and repository["repository_study_ID"] + ): + repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[ + repository["repository_name"] + ].replace("", repository["repository_study_ID"]) + repository.update({"repository_study_link": repository_study_link}) + if repository_citation_additional_text not in repository_citation: + repository_citation += repository_citation_additional_text + if len(data_repositories): + data_repositories[0] = { + **data_repositories[0], + "repository_citation": repository_citation, + } + + mds_res["gen3_discovery"]["study_metadata"][ + "metadata_location" + ]["data_repositories"] = copy.deepcopy(data_repositories) + + + + # set up related studies + serial_num = None + try: + serial_num = ( + mds_res + .get("nih_reporter", {}) + .get("project_num_split", {}) + .get("serial_num", None) + ) + except Exception: + print(f"Unable to get serial number for study") + + if serial_num == None: + print(f"Unable to get serial number for study") + + related_study_result = get_related_studies(serial_num, hostname) + existing_related_study_result = mds_res.get("related_studies", []) + for related_study in related_study_result: + if related_study not in existing_related_study_result: + existing_related_study_result.append(copy.deepcopy(related_study)) + mds_res["gen3_discovery"][ + "related_studies" + ] = copy.deepcopy(existing_related_study_result) + + # merge data from cedar that is not study level metadata into a level higher deleted_keys = [] for key, value in mds_res["gen3_discovery"]["study_metadata"].items(): From 99fc77ac5ee4a36443a9b802d903e6ffaab6f8c7 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 7 Mar 2024 17:32:40 -0600 Subject: [PATCH 077/114] MIDRC-602 Add ecr-access job (#2480) --- .pre-commit-config.yaml | 2 +- .secrets.baseline | 3792 +++++++++++++---- files/scripts/ecr-access-job-requirements.txt | 1 + files/scripts/ecr-access-job.md | 85 + files/scripts/ecr-access-job.py | 177 + gen3/bin/kube-setup-ecr-access-cronjob.sh | 61 + kube/services/jobs/ecr-access-job.yaml | 85 + 7 files changed, 3339 insertions(+), 864 deletions(-) create mode 100644 files/scripts/ecr-access-job-requirements.txt create mode 100644 files/scripts/ecr-access-job.md create mode 100644 files/scripts/ecr-access-job.py create mode 100644 gen3/bin/kube-setup-ecr-access-cronjob.sh create mode 100644 kube/services/jobs/ecr-access-job.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e3ce795b6..82034495d3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: git@github.com:Yelp/detect-secrets - rev: v0.13.1 + rev: v1.4.0 hooks: - id: detect-secrets args: ['--baseline', '.secrets.baseline'] diff --git a/.secrets.baseline b/.secrets.baseline index 200b69841f..0c4eba0a80 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1,19 +1,18 @@ { - "exclude": { - "files": null, - "lines": null - }, - "generated_at": "2024-03-04T21:42:56Z", + "version": "1.4.0", "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, { "name": "AWSKeyDetector" }, { - "name": "ArtifactoryDetector" + "name": "AzureStorageKeyDetector" }, { - "base64_limit": 4.5, - "name": "Base64HighEntropyString" + "name": "Base64HighEntropyString", + "limit": 4.5 }, { "name": "BasicAuthDetector" @@ -22,8 +21,14 @@ "name": "CloudantDetector" }, { - "hex_limit": 3, - "name": "HexHighEntropyString" + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 }, { "name": "IbmCloudIamDetector" @@ -35,21 +40,30 @@ "name": "JwtTokenDetector" }, { - "keyword_exclude": null, - "name": "KeywordDetector" + "name": "KeywordDetector", + "keyword_exclude": "" }, { "name": "MailchimpDetector" }, + { + "name": "NpmDetector" + }, { "name": "PrivateKeyDetector" }, + { + "name": "SendGridDetector" + }, { "name": "SlackDetector" }, { "name": "SoftlayerDetector" }, + { + "name": "SquareOAuthDetector" + }, { "name": "StripeDetector" }, @@ -57,1619 +71,3671 @@ "name": "TwilioKeyDetector" } ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], "results": { "Chef/repo/data_bags/README.md": [ { - "hashed_secret": "8a9250639e092d90f164792e35073a9395bff366", - "is_verified": false, - "line_number": 45, - "type": "Secret Keyword" - }, - { + "type": "Secret Keyword", + "filename": "Chef/repo/data_bags/README.md", "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee", "is_verified": false, - "line_number": 51, - "type": "Secret Keyword" + "line_number": 38 } ], - "Docker/jenkins/Jenkins-CI-Worker/Dockerfile": [ + "Docker/sidecar/service.key": [ { - "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "type": "Private Key", + "filename": "Docker/sidecar/service.key", + "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", "is_verified": false, - "line_number": 124, - "type": "Secret Keyword" + "line_number": 1 } ], - "Docker/jenkins/Jenkins-Worker/Dockerfile": [ + "Jenkins/Stacks/Jenkins/jenkins.env.sample": [ { - "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "type": "Secret Keyword", + "filename": "Jenkins/Stacks/Jenkins/jenkins.env.sample", + "hashed_secret": "f41a52528dd2d592d2c05de5f388101c2948aa98", "is_verified": false, - "line_number": 139, - "type": "Secret Keyword" + "line_number": 5 } ], - "Docker/jenkins/Jenkins/Dockerfile": [ + "Jenkinsfile": [ { - "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "type": "Secret Keyword", + "filename": "Jenkinsfile", + "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf", "is_verified": false, - "line_number": 107, - "type": "Secret Keyword" - } - ], - "Docker/jenkins/Jenkins2/Dockerfile": [ + "line_number": 144 + }, { - "hashed_secret": "10daf3a26c6a17242a5ab2438a12ebc8276c7603", + "type": "Secret Keyword", + "filename": "Jenkinsfile", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 108, - "type": "Secret Keyword" + "line_number": 147 } ], - "Docker/sidecar/service.key": [ + "ansible/roles/slurm/README.md": [ { - "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9", + "type": "Base64 High Entropy String", + "filename": "ansible/roles/slurm/README.md", + "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42", "is_verified": false, - "line_number": 1, - "type": "Private Key" + "line_number": 86 } ], - "Jenkins/Stacks/Jenkins/jenkins.env.sample": [ + "apis_configs/fence_settings.py": [ { - "hashed_secret": "eecee33686ac5861c2a7edc8b46bd0e5432bfddd", + "type": "Basic Auth Credentials", + "filename": "apis_configs/fence_settings.py", + "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", "is_verified": false, - "line_number": 5, - "type": "Secret Keyword" + "line_number": 80 } ], - "ansible/roles/awslogs/defaults/main.yaml": [ + "apis_configs/peregrine_settings.py": [ { - "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684", + "type": "Basic Auth Credentials", + "filename": "apis_configs/peregrine_settings.py", + "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", "is_verified": false, - "line_number": 30, - "type": "Basic Auth Credentials" + "line_number": 46 } ], - "ansible/roles/slurm/README.md": [ - { - "hashed_secret": "4acfde1ff9c353ba2ef0dbe0df73bda2743cba42", - "is_verified": false, - "line_number": 86, - "type": "Base64 High Entropy String" - }, + "apis_configs/sheepdog_settings.py": [ { - "hashed_secret": "579649582303921502d9e6d3f8755f13fdd2b476", + "type": "Basic Auth Credentials", + "filename": "apis_configs/sheepdog_settings.py", + "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", "is_verified": false, - "line_number": 86, - "type": "Secret Keyword" + "line_number": 46 } ], - "apis_configs/config_helper.py": [ + "aws-inspec/kubernetes/chef_inspec-cron.yaml": [ { - "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", + "type": "Secret Keyword", + "filename": "aws-inspec/kubernetes/chef_inspec-cron.yaml", + "hashed_secret": "a3ba27250861948a554629a0e21168821ddfa9f1", "is_verified": false, - "line_number": 66, - "type": "Basic Auth Credentials" + "line_number": 35 } ], - "apis_configs/fence_credentials.json": [ + "doc/api.md": [ { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "type": "Hex High Entropy String", + "filename": "doc/api.md", + "hashed_secret": "625de83a7517422051911680cc803921ff99db90", "is_verified": false, - "line_number": 23, - "type": "Secret Keyword" + "line_number": 47 } ], - "apis_configs/fence_settings.py": [ + "doc/gen3OnK8s.md": [ { - "hashed_secret": "3ef0fb8a603abdc0b6caac44a23fdc6792f77ddf", + "type": "Secret Keyword", + "filename": "doc/gen3OnK8s.md", + "hashed_secret": "55c100ba37d2df35ec1e5f5d6302f060387df6cc", "is_verified": false, - "line_number": 6, - "type": "Basic Auth Credentials" + "line_number": 113 }, { - "hashed_secret": "b60d121b438a380c343d5ec3c2037564b82ffef3", + "type": "Secret Keyword", + "filename": "doc/gen3OnK8s.md", + "hashed_secret": "262d8e9b8ac5f06e7612dfb608f7267f88679801", "is_verified": false, - "line_number": 58, - "type": "Secret Keyword" + "line_number": 120 }, { - "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "type": "Secret Keyword", + "filename": "doc/gen3OnK8s.md", + "hashed_secret": "1c17e556736c4d23933f99d199e7c2c572895fd2", + "is_verified": false, + "line_number": 143 + }, + { + "type": "Secret Keyword", + "filename": "doc/gen3OnK8s.md", + "hashed_secret": "76a4acaf31b815aa2c41cc2a2176b11fa9edf00a", + "is_verified": false, + "line_number": 145 + }, + { + "type": "Secret Keyword", + "filename": "doc/gen3OnK8s.md", + "hashed_secret": "9d678cbce5a343920f754d5836f03346ee01cde5", "is_verified": false, - "line_number": 80, - "type": "Basic Auth Credentials" + "line_number": 154 } ], - "apis_configs/indexd_settings.py": [ + "files/scripts/psql-fips-fix.sh": [ { - "hashed_secret": "0a0d18c85e096611b5685b62bc60ec534d19bacc", + "type": "Secret Keyword", + "filename": "files/scripts/psql-fips-fix.sh", + "hashed_secret": "2f1aa1e2a58704b452a5dd60ab1bd2b761bf296a", "is_verified": false, - "line_number": 59, - "type": "Basic Auth Credentials" + "line_number": 9 } ], - "apis_configs/peregrine_settings.py": [ + "gen3/bin/bucket-manifest.sh": [ { - "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "type": "Secret Keyword", + "filename": "gen3/bin/bucket-manifest.sh", + "hashed_secret": "2be88ca4242c76e8253ac62474851065032d6833", "is_verified": false, - "line_number": 46, - "type": "Basic Auth Credentials" + "line_number": 58 } ], - "apis_configs/sheepdog_settings.py": [ + "gen3/bin/bucket-replicate.sh": [ { - "hashed_secret": "347cd9c53ff77d41a7b22aa56c7b4efaf54658e3", + "type": "Secret Keyword", + "filename": "gen3/bin/bucket-replicate.sh", + "hashed_secret": "2be88ca4242c76e8253ac62474851065032d6833", "is_verified": false, - "line_number": 46, - "type": "Basic Auth Credentials" + "line_number": 39 } ], - "doc/Gen3-data-upload.md": [ + "gen3/bin/secrets.sh": [ { - "hashed_secret": "b8bd20d4a2701dc3aba0efbbf325f1359392d93e", + "type": "Secret Keyword", + "filename": "gen3/bin/secrets.sh", + "hashed_secret": "fb6220478aaba649aac37271a1d7c6317abc03a6", "is_verified": false, - "line_number": 26, - "type": "Secret Keyword" + "line_number": 135 } ], - "doc/api.md": [ + "gen3/lib/aws.sh": [ { - "hashed_secret": "625de83a7517422051911680cc803921ff99db90", + "type": "Secret Keyword", + "filename": "gen3/lib/aws.sh", + "hashed_secret": "6b44a330b450ee550c081410c6b705dfeaa105ce", "is_verified": false, - "line_number": 47, - "type": "Hex High Entropy String" + "line_number": 640 } ], - "doc/gen3OnK8s.md": [ + "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [ { - "hashed_secret": "2db6d21d365f544f7ca3bcfb443ac96898a7a069", + "type": "Basic Auth Credentials", + "filename": "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml", + "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", "is_verified": false, - "line_number": 113, - "type": "Secret Keyword" - }, + "line_number": 33 + } + ], + "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json": [ { - "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2", + "type": "Secret Keyword", + "filename": "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json", + "hashed_secret": "0447a636536df0264b2000403fbefd69f603ceb1", "is_verified": false, - "line_number": 143, - "type": "Secret Keyword" + "line_number": 54 }, { - "hashed_secret": "70374248fd7129088fef42b8f568443f6dce3a48", + "type": "Secret Keyword", + "filename": "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json", + "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966", "is_verified": false, - "line_number": 170, - "type": "Secret Keyword" + "line_number": 60 }, { - "hashed_secret": "bcf22dfc6fb76b7366b1f1675baf2332a0e6a7ce", + "type": "Secret Keyword", + "filename": "gen3/lib/bootstrap/templates/cdis-manifest/manifests/sower/sower.json", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 189, - "type": "Secret Keyword" + "line_number": 108 } ], - "doc/kube-setup-data-ingestion-job.md": [ + "gen3/lib/onprem.sh": [ { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "type": "Secret Keyword", + "filename": "gen3/lib/onprem.sh", + "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3", "is_verified": false, - "line_number": 30, - "type": "Secret Keyword" - } - ], - "doc/logs.md": [ + "line_number": 68 + }, { - "hashed_secret": "9addbf544119efa4a64223b649750a510f0d463f", + "type": "Secret Keyword", + "filename": "gen3/lib/onprem.sh", + "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70", "is_verified": false, - "line_number": 6, - "type": "Secret Keyword" + "line_number": 84 } ], - "doc/slurm_cluster.md": [ + "gen3/lib/testData/default/expectedFenceResult.yaml": [ { - "hashed_secret": "2ace62c1befa19e3ea37dd52be9f6d508c5163e6", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 184, - "type": "Secret Keyword" - } - ], - "files/dashboard/usage-reports/package-lock.json": [ + "line_number": 68 + }, { - "hashed_secret": "e095101882f706c4de95e0f75c5bcb9666e3f448", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 10, - "type": "Base64 High Entropy String" + "line_number": 71 }, { - "hashed_secret": "5422e4f96964d5739998b25ac214520c1b113e5b", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 15, - "type": "Base64 High Entropy String" - } - ], - "gen3/bin/api.sh": [ + "line_number": 74 + }, { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 407, - "type": "Secret Keyword" + "line_number": 84 }, { - "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 477, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-dev-namespace.sh": [ + "line_number": 87 + }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 135, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-setup-argo.sh": [ + "line_number": 90 + }, { - "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", "is_verified": false, - "line_number": 206, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-setup-aurora-monitoring.sh": [ + "line_number": 93 + }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", "is_verified": false, - "line_number": 59, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-setup-certs.sh": [ + "line_number": 96 + }, { - "hashed_secret": "2e9ee120fd25e31048598693aca91d5473898a99", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedFenceResult.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 50, - "type": "Secret Keyword" + "line_number": 99 } ], - "gen3/bin/kube-setup-dashboard.sh": [ + "gen3/lib/testData/default/expectedSheepdogResult.yaml": [ { - "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml", + "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d", "is_verified": false, - "line_number": 40, - "type": "Secret Keyword" + "line_number": 60 }, { - "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 41, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-setup-data-ingestion-job.sh": [ + "line_number": 63 + }, { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml", + "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2", "is_verified": false, - "line_number": 37, - "type": "Secret Keyword" + "line_number": 69 }, { - "hashed_secret": "8695a632956b1b0ea7b66993dcc98732da39148c", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/default/expectedSheepdogResult.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 102, - "type": "Secret Keyword" + "line_number": 72 } ], - "gen3/bin/kube-setup-dicom-server.sh": [ + "gen3/lib/testData/etlconvert/expected2.yaml": [ { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/expected2.yaml", + "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474", "is_verified": false, - "line_number": 43, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-setup-dicom.sh": [ + "line_number": 10 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/expected2.yaml", + "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55", + "is_verified": false, + "line_number": 13 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/expected2.yaml", + "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26", + "is_verified": false, + "line_number": 16 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/expected2.yaml", + "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12", + "is_verified": false, + "line_number": 18 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/expected2.yaml", + "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424", + "is_verified": false, + "line_number": 33 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/expected2.yaml", + "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba", + "is_verified": false, + "line_number": 35 + }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/expected2.yaml", + "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22", "is_verified": false, - "line_number": 78, - "type": "Secret Keyword" + "line_number": 36 } ], - "gen3/bin/kube-setup-gen3-discovery-ai.sh": [ + "gen3/lib/testData/etlconvert/users2.yaml": [ + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/users2.yaml", + "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55", + "is_verified": false, + "line_number": 543 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/users2.yaml", + "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26", + "is_verified": false, + "line_number": 553 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/users2.yaml", + "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474", + "is_verified": false, + "line_number": 558 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/users2.yaml", + "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12", + "is_verified": false, + "line_number": 568 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/users2.yaml", + "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424", + "is_verified": false, + "line_number": 643 + }, { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/users2.yaml", + "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba", "is_verified": false, - "line_number": 37, - "type": "Secret Keyword" + "line_number": 653 }, { - "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "type": "Base64 High Entropy String", + "filename": "gen3/lib/testData/etlconvert/users2.yaml", + "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22", "is_verified": false, - "line_number": 71, - "type": "Secret Keyword" + "line_number": 658 } ], - "gen3/bin/kube-setup-jenkins.sh": [ + "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml": [ + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 71 + }, + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 74 + }, + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 77 + }, + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 87 + }, { - "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 18, - "type": "Secret Keyword" + "line_number": 90 }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 22, - "type": "Secret Keyword" + "line_number": 93 + }, + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 96 + }, + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", + "is_verified": false, + "line_number": 99 + }, + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedFenceResult.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 102 } ], - "gen3/bin/kube-setup-jenkins2.sh": [ + "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml": [ + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml", + "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d", + "is_verified": false, + "line_number": 63 + }, { - "hashed_secret": "05ea760643a5c0a9bacb3544dc844ac79938a51f", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 18, - "type": "Secret Keyword" + "line_number": 66 }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml", + "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2", "is_verified": false, - "line_number": 22, - "type": "Secret Keyword" + "line_number": 72 + }, + { + "type": "Secret Keyword", + "filename": "gen3/lib/testData/test1.manifest.g3k/expectedSheepdogResult.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 75 } ], - "gen3/bin/kube-setup-metadata.sh": [ + "gen3/test/secretsTest.sh": [ { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "gen3/test/secretsTest.sh", + "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938", "is_verified": false, - "line_number": 35, - "type": "Secret Keyword" + "line_number": 25 } ], - "gen3/bin/kube-setup-revproxy.sh": [ + "gen3/test/terraformTest.sh": [ { - "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "6b44a330b450ee550c081410c6b705dfeaa105ce", "is_verified": false, - "line_number": 38, - "type": "Secret Keyword" + "line_number": 156 }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "d869db7fe62fb07c25a0403ecaea55031744b5fb", "is_verified": false, - "line_number": 55, - "type": "Secret Keyword" + "line_number": 163 }, { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Base64 High Entropy String", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009", "is_verified": false, - "line_number": 57, - "type": "Secret Keyword" - } - ], - "gen3/bin/kube-setup-secrets.sh": [ + "line_number": 172 + }, + { + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009", + "is_verified": false, + "line_number": 172 + }, + { + "type": "Base64 High Entropy String", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef", + "is_verified": false, + "line_number": 175 + }, + { + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef", + "is_verified": false, + "line_number": 175 + }, + { + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "212e1d3823c8c9af9e4c0c172164ee292b9a6768", + "is_verified": false, + "line_number": 311 + }, { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "cb80dbb67a1a5bdf4957eea1473789f1c65357c6", "is_verified": false, - "line_number": 79, - "type": "Secret Keyword" + "line_number": 312 }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "5f35c25f4bf588b5fad46e249fcd9221f5257ce4", "is_verified": false, - "line_number": 82, - "type": "Secret Keyword" + "line_number": 313 }, { - "hashed_secret": "6f7531b95bbc99ac25a5cc82edb825f319c5dee8", + "type": "Secret Keyword", + "filename": "gen3/test/terraformTest.sh", + "hashed_secret": "5308421b43dde5775f1993bd25a8163070d65598", "is_verified": false, - "line_number": 95, - "type": "Secret Keyword" + "line_number": 314 } ], - "gen3/bin/kube-setup-sftp.sh": [ + "kube/services/access-backend/access-backend-deploy.yaml": [ { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "kube/services/access-backend/access-backend-deploy.yaml", + "hashed_secret": "dbf88a0c3d905c669c0fd13bf8172bb34d4b1168", "is_verified": false, - "line_number": 36, - "type": "Secret Keyword" - }, + "line_number": 60 + } + ], + "kube/services/acronymbot/acronymbot-deploy.yaml": [ { - "hashed_secret": "83d11e3aec005a3b9a2077c6800683e202a95af4", + "type": "Secret Keyword", + "filename": "kube/services/acronymbot/acronymbot-deploy.yaml", + "hashed_secret": "600833390a6b9891d0d8a5f6e3326abb237ac8ca", "is_verified": false, - "line_number": 51, - "type": "Secret Keyword" + "line_number": 49 } ], - "gen3/bin/kube-setup-sheepdog.sh": [ + "kube/services/arborist/arborist-deploy-2.yaml": [ { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy-2.yaml", + "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab", "is_verified": false, - "line_number": 33, - "type": "Secret Keyword" + "line_number": 59 } ], - "gen3/bin/kube-setup-sower-jobs.sh": [ + "kube/services/arborist/arborist-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 64 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 67 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 70 + }, { - "hashed_secret": "40304f287a52d99fdbe086ad19dbdbf9cc1b3897", + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 25, - "type": "Secret Keyword" + "line_number": 77 }, { - "hashed_secret": "e7064f0b80f61dbc65915311032d27baa569ae2a", + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 26, - "type": "Secret Keyword" + "line_number": 80 }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 120, - "type": "Secret Keyword" + "line_number": 83 }, { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "kube/services/arborist/arborist-deploy.yaml", + "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73", "is_verified": false, - "line_number": 122, - "type": "Secret Keyword" + "line_number": 86 } ], - "gen3/bin/kube-setup-ssjdispatcher.sh": [ + "kube/services/argo/workflows/fence-usersync-wf.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 108 + }, { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 117, - "type": "Secret Keyword" + "line_number": 111 }, { - "hashed_secret": "7992309146efaa8da936e34b0bd33242cd0e9f93", + "type": "Secret Keyword", + "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 184, - "type": "Secret Keyword" + "line_number": 114 }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 197, - "type": "Secret Keyword" + "line_number": 117 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/argo/workflows/fence-usersync-wf.yaml", + "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73", + "is_verified": false, + "line_number": 120 } ], - "gen3/lib/aws.sh": [ + "kube/services/argocd/values.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/argocd/values.yaml", + "hashed_secret": "bfc1b86ce643b65bd540989213254b01fd6ad418", + "is_verified": false, + "line_number": 1489 + } + ], + "kube/services/arranger/arranger-deploy.yaml": [ { - "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", + "type": "Secret Keyword", + "filename": "kube/services/arranger/arranger-deploy.yaml", + "hashed_secret": "0db22b31c9add2d3c76743c0ac6fbc99bb8b4761", "is_verified": false, - "line_number": 640, - "type": "Secret Keyword" + "line_number": 61 }, { - "hashed_secret": "5b4b6c62d3d99d202f095c38c664eded8f640ce8", + "type": "Secret Keyword", + "filename": "kube/services/arranger/arranger-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 660, - "type": "Secret Keyword" + "line_number": 64 } ], - "gen3/lib/bootstrap/templates/Gen3Secrets/apis_configs/fence-config.yaml": [ + "kube/services/audit-service/audit-service-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/audit-service/audit-service-deploy.yaml", + "hashed_secret": "42cde1c58c36d8bb5804a076e55ac6ec07ef99fc", + "is_verified": false, + "line_number": 64 + } + ], + "kube/services/aws-es-proxy/aws-es-proxy-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/aws-es-proxy/aws-es-proxy-deploy.yaml", + "hashed_secret": "7f834ccb442433fc12ec9532f75c3a4b6a748d4c", + "is_verified": false, + "line_number": 46 + } + ], + "kube/services/cedar-wrapper/cedar-wrapper-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/cedar-wrapper/cedar-wrapper-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 56 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/cedar-wrapper/cedar-wrapper-deploy.yaml", + "hashed_secret": "5949b79e0c7082dc78d543cde662871a4f8b8913", + "is_verified": false, + "line_number": 59 + } + ], + "kube/services/cogwheel/cogwheel-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/cogwheel/cogwheel-deploy.yaml", + "hashed_secret": "09b772df628fd10bca646b6a877eb661122210ab", + "is_verified": false, + "line_number": 35 + } + ], + "kube/services/cohort-middleware/cohort-middleware-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/cohort-middleware/cohort-middleware-deploy.yaml", + "hashed_secret": "bf22f6c4bd03572f1ef593efc3eb1a7e0b6dcab4", + "is_verified": false, + "line_number": 62 + } + ], + "kube/services/dashboard/dashboard-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/dashboard/dashboard-deploy.yaml", + "hashed_secret": "9e722d12ce045c8718ab803ed465b2fbe199f3d3", + "is_verified": false, + "line_number": 61 + } + ], + "kube/services/datadog/values.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/datadog/values.yaml", + "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b", + "is_verified": false, + "line_number": 23 + } + ], + "kube/services/datasim/datasim-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/datasim/datasim-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 63 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/datasim/datasim-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 66 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/datasim/datasim-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 72 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/datasim/datasim-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 76 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/datasim/datasim-deploy.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 79 + } + ], + "kube/services/dicom-server/dicom-server-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/dicom-server/dicom-server-deploy.yaml", + "hashed_secret": "706168ac2565a93cceffe2202ac45d3d31c075fb", + "is_verified": false, + "line_number": 40 + } + ], + "kube/services/fence/fence-canary-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 68 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 71 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 74 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 84 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 87 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 90 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 93 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", + "is_verified": false, + "line_number": 96 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-canary-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 99 + } + ], + "kube/services/fence/fence-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 71 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 74 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 77 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 87 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 90 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 93 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 96 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", + "is_verified": false, + "line_number": 99 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fence/fence-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 102 + } + ], + "kube/services/fenceshib/fenceshib-canary-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 62 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 65 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 68 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 78 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 81 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 84 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 87 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", + "is_verified": false, + "line_number": 90 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-canary-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 93 + } + ], + "kube/services/fenceshib/fenceshib-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 69 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 72 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 75 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 85 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 88 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 91 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 94 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", + "is_verified": false, + "line_number": 97 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 100 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/fenceshib/fenceshib-deploy.yaml", + "hashed_secret": "6c4789c3be186fd5dcbf06723462ccdd2c86dc37", + "is_verified": false, + "line_number": 103 + } + ], + "kube/services/frontend-framework/frontend-framework-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 54 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml", + "hashed_secret": "6607b403f74e62246fc6a3c938feffc5a34a7e49", + "is_verified": false, + "line_number": 57 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml", + "hashed_secret": "4b0bb3e58651fe56ee23e59aa6a3cb96dc61ddd2", + "is_verified": false, + "line_number": 60 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-deploy.yaml", + "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9", + "is_verified": false, + "line_number": 66 + } + ], + "kube/services/frontend-framework/frontend-framework-root-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 54 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml", + "hashed_secret": "6607b403f74e62246fc6a3c938feffc5a34a7e49", + "is_verified": false, + "line_number": 57 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml", + "hashed_secret": "4b0bb3e58651fe56ee23e59aa6a3cb96dc61ddd2", + "is_verified": false, + "line_number": 60 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/frontend-framework/frontend-framework-root-deploy.yaml", + "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9", + "is_verified": false, + "line_number": 66 + } + ], + "kube/services/gdcapi/gdcapi-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/gdcapi/gdcapi-deploy.yaml", + "hashed_secret": "e8c2f0bacaffbf2f9897217c6770413879945296", + "is_verified": false, + "line_number": 38 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/gdcapi/gdcapi-deploy.yaml", + "hashed_secret": "517cded9f3e3ab79237fde330b97a93f5a943316", + "is_verified": false, + "line_number": 41 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/gdcapi/gdcapi-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 44 + } + ], + "kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/gen3-discovery-ai/gen3-discovery-ai-deploy.yaml", + "hashed_secret": "38ded89f83435a558169dedb91a38f72d6cebf41", + "is_verified": false, + "line_number": 27 + } + ], + "kube/services/google-sa-validation/google-sa-validation-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 54 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 57 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 63 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 67 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 70 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 73 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 76 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", + "is_verified": false, + "line_number": 79 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/google-sa-validation/google-sa-validation-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 82 + } + ], + "kube/services/guppy/guppy-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/guppy/guppy-deploy.yaml", + "hashed_secret": "0db22b31c9add2d3c76743c0ac6fbc99bb8b4761", + "is_verified": false, + "line_number": 65 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/guppy/guppy-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 68 + } + ], + "kube/services/indexd/indexd-canary-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-canary-deploy.yaml", + "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3", + "is_verified": false, + "line_number": 59 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-canary-deploy.yaml", + "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039", + "is_verified": false, + "line_number": 62 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-canary-deploy.yaml", + "hashed_secret": "bdecca54d39013d43d3b7f05f2927eaa7df375dc", + "is_verified": false, + "line_number": 68 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-canary-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 71 + } + ], + "kube/services/indexd/indexd-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-deploy.yaml", + "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3", + "is_verified": false, + "line_number": 63 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-deploy.yaml", + "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039", + "is_verified": false, + "line_number": 66 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-deploy.yaml", + "hashed_secret": "bdecca54d39013d43d3b7f05f2927eaa7df375dc", + "is_verified": false, + "line_number": 72 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/indexd/indexd-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 75 + } + ], + "kube/services/jenkins-ci-worker/jenkins-ci-worker-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins-ci-worker/jenkins-ci-worker-deploy.yaml", + "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf", + "is_verified": false, + "line_number": 143 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins-ci-worker/jenkins-ci-worker-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 146 + } + ], + "kube/services/jenkins-worker/jenkins-worker-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins-worker/jenkins-worker-deploy.yaml", + "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf", + "is_verified": false, + "line_number": 150 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins-worker/jenkins-worker-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 153 + } + ], + "kube/services/jenkins/jenkins-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins/jenkins-deploy.yaml", + "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf", + "is_verified": false, + "line_number": 157 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins/jenkins-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 160 + } + ], + "kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml", + "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf", + "is_verified": false, + "line_number": 143 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins2-ci-worker/jenkins2-ci-worker-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 146 + } + ], + "kube/services/jenkins2-worker/jenkins2-worker-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins2-worker/jenkins2-worker-deploy.yaml", + "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf", + "is_verified": false, + "line_number": 146 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins2-worker/jenkins2-worker-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 149 + } + ], + "kube/services/jenkins2/jenkins2-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins2/jenkins2-deploy.yaml", + "hashed_secret": "c937b6fbb346a51ef679dd02ac5c4863e02bfdbf", + "is_verified": false, + "line_number": 153 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jenkins2/jenkins2-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 156 + } + ], + "kube/services/jobs/arborist-rm-expired-access-cronjob.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/arborist-rm-expired-access-cronjob.yaml", + "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab", + "is_verified": false, + "line_number": 37 + } + ], + "kube/services/jobs/arborist-rm-expired-access-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/arborist-rm-expired-access-job.yaml", + "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab", + "is_verified": false, + "line_number": 37 + } + ], + "kube/services/jobs/arboristdb-create-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/arboristdb-create-job.yaml", + "hashed_secret": "6c57cdfdaaf3cde7a1da6aa94c7d8e46502c4bab", + "is_verified": false, + "line_number": 33 + } + ], + "kube/services/jobs/aws-bucket-replicate-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/aws-bucket-replicate-job.yaml", + "hashed_secret": "deb02468778f4041fb189654698ac948e436732d", + "is_verified": false, + "line_number": 33 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/aws-bucket-replicate-job.yaml", + "hashed_secret": "abe72fcb190ed9c73eb20e198c73a97605b95063", + "is_verified": false, + "line_number": 36 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/aws-bucket-replicate-job.yaml", + "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb", + "is_verified": false, + "line_number": 42 + } + ], + "kube/services/jobs/bucket-manifest-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/bucket-manifest-job.yaml", + "hashed_secret": "6c36710fe8825b381388d7005f2c9b5c70175fba", + "is_verified": false, + "line_number": 33 + } + ], + "kube/services/jobs/bucket-replicate-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/bucket-replicate-job.yaml", + "hashed_secret": "84954f7729144580d612cbb0517aeca8880e3483", + "is_verified": false, + "line_number": 46 + } + ], + "kube/services/jobs/bucket-replication-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/bucket-replication-job.yaml", + "hashed_secret": "84954f7729144580d612cbb0517aeca8880e3483", + "is_verified": false, + "line_number": 32 + } + ], + "kube/services/jobs/bucket-size-report-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/bucket-size-report-job.yaml", + "hashed_secret": "7cccf62cb63863d9d3baabed4f576eb0f7039735", + "is_verified": false, + "line_number": 34 + } + ], + "kube/services/jobs/cedar-ingestion-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/cedar-ingestion-job.yaml", + "hashed_secret": "e1c426d126dcc618dcd0686fc718d509ca6ee3b8", + "is_verified": false, + "line_number": 54 + } + ], + "kube/services/jobs/client-modify-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/client-modify-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 41 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/client-modify-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 44 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/client-modify-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 50 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/client-modify-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 54 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/client-modify-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 57 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/client-modify-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 60 + } + ], + "kube/services/jobs/cogwheel-register-client-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/cogwheel-register-client-job.yaml", + "hashed_secret": "09b772df628fd10bca646b6a877eb661122210ab", + "is_verified": false, + "line_number": 40 + } + ], + "kube/services/jobs/config-fence-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/config-fence-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 44 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/config-fence-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 54 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/config-fence-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 57 + } + ], + "kube/services/jobs/covid19-etl-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/covid19-etl-job.yaml", + "hashed_secret": "a7a2b42615b2b256a7c601c77c426e5d6cafb212", + "is_verified": false, + "line_number": 34 + } + ], + "kube/services/jobs/covid19-notebook-etl-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/covid19-notebook-etl-job.yaml", + "hashed_secret": "a7a2b42615b2b256a7c601c77c426e5d6cafb212", + "is_verified": false, + "line_number": 33 + } + ], + "kube/services/jobs/data-ingestion-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/data-ingestion-job.yaml", + "hashed_secret": "81e4388059839f71aed21999aa51095c7e545094", + "is_verified": false, + "line_number": 34 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/data-ingestion-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 48 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/data-ingestion-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 51 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/data-ingestion-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 54 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/data-ingestion-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 60 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/data-ingestion-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 63 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/data-ingestion-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 66 + } + ], + "kube/services/jobs/etl-cronjob.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/etl-cronjob.yaml", + "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966", + "is_verified": false, + "line_number": 38 + } + ], + "kube/services/jobs/etl-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/etl-job.yaml", + "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966", + "is_verified": false, + "line_number": 35 + } + ], + "kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 43 + } + ], + "kube/services/jobs/fence-cleanup-expired-ga4gh-info-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-cleanup-expired-ga4gh-info-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 36 + } + ], + "kube/services/jobs/fence-db-migrate-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-db-migrate-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 36 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-db-migrate-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 39 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-db-migrate-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 42 + } + ], + "kube/services/jobs/fence-delete-expired-clients-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-delete-expired-clients-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 38 + } + ], + "kube/services/jobs/fence-visa-update-cronjob.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-visa-update-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 42 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-visa-update-cronjob.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 45 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-visa-update-cronjob.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 48 + } + ], + "kube/services/jobs/fence-visa-update-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-visa-update-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 36 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-visa-update-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 39 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fence-visa-update-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 42 + } + ], + "kube/services/jobs/fencedb-create-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/fencedb-create-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 33 + } + ], + "kube/services/jobs/gdcdb-create-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gdcdb-create-job.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", + "is_verified": false, + "line_number": 33 + } + ], + "kube/services/jobs/gen3qa-check-bucket-access-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 177 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 180 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 186 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 190 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gen3qa-check-bucket-access-job.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 193 + } + ], + "kube/services/jobs/gentestdata-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gentestdata-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 67 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gentestdata-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 70 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gentestdata-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 76 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gentestdata-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 80 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/gentestdata-job.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", + "is_verified": false, + "line_number": 83 + } + ], + "kube/services/jobs/google-bucket-manifest-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-bucket-manifest-job.yaml", + "hashed_secret": "5ca8fff7767e5dd6ebed80e2c8eab66d6f3bf5eb", + "is_verified": false, + "line_number": 31 + } + ], + "kube/services/jobs/google-bucket-replicate-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-bucket-replicate-job.yaml", + "hashed_secret": "b6f0ec0b08da77656ced48427841e28d7a8a81d6", + "is_verified": false, + "line_number": 35 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-bucket-replicate-job.yaml", + "hashed_secret": "abe72fcb190ed9c73eb20e198c73a97605b95063", + "is_verified": false, + "line_number": 38 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-bucket-replicate-job.yaml", + "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb", + "is_verified": false, + "line_number": 41 + } + ], + "kube/services/jobs/google-create-bucket-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-create-bucket-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 78 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-create-bucket-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 81 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-create-bucket-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 84 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-create-bucket-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 91 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-create-bucket-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 94 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-create-bucket-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 97 + } + ], + "kube/services/jobs/google-delete-expired-access-cronjob.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-access-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 43 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-access-cronjob.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 46 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-access-cronjob.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 49 + } + ], + "kube/services/jobs/google-delete-expired-access-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-access-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 36 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-access-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 39 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-access-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 42 + } + ], + "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 48 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 51 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 57 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 61 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 64 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-cronjob.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", + "is_verified": false, + "line_number": 67 + } + ], + "kube/services/jobs/google-delete-expired-service-account-job.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", + "is_verified": false, + "line_number": 40 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", + "is_verified": false, + "line_number": 43 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", + "is_verified": false, + "line_number": 49 + }, { - "hashed_secret": "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 33, - "type": "Basic Auth Credentials" + "line_number": 53 }, { - "hashed_secret": "5d07e1b80e448a213b392049888111e1779a52db", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 286, - "type": "Secret Keyword" - } - ], - "gen3/lib/bootstrap/templates/Gen3Secrets/creds.json": [ + "line_number": 56 + }, { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-delete-expired-service-account-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 26, - "type": "Secret Keyword" + "line_number": 59 } ], - "gen3/lib/bootstrap/templates/Gen3Secrets/g3auto/dbfarm/servers.json": [ + "kube/services/jobs/google-init-proxy-groups-cronjob.yaml": [ { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 5, - "type": "Secret Keyword" - } - ], - "gen3/lib/logs/utils.sh": [ + "line_number": 48 + }, { - "hashed_secret": "76143b4ffc8aa2a53f9700ce229f904e69f1e8b5", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 3, - "type": "Secret Keyword" - } - ], - "gen3/lib/manifestDefaults/hatchery/hatchery.json": [ + "line_number": 51 + }, { - "hashed_secret": "0da0e0005ca04acb407af2681d0bede6d9406039", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 78, - "type": "Secret Keyword" - } - ], - "gen3/lib/onprem.sh": [ + "line_number": 54 + }, { - "hashed_secret": "29e52a9bac8f274fa41c51fce9c98eba0dd99cb3", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 68, - "type": "Secret Keyword" + "line_number": 61 }, { - "hashed_secret": "50f013532a9770a2c2cfdc38b7581dd01df69b70", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 84, - "type": "Secret Keyword" - } - ], - "gen3/lib/secrets/rotate-postgres.sh": [ + "line_number": 64 + }, { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 162, - "type": "Secret Keyword" + "line_number": 67 }, { - "hashed_secret": "d3df8a3b08a9de43b73eca1302d50e7a0e5b360f", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-cronjob.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", "is_verified": false, - "line_number": 250, - "type": "Secret Keyword" + "line_number": 70 } ], - "gen3/lib/testData/etlconvert/expected2.yaml": [ + "kube/services/jobs/google-init-proxy-groups-job.yaml": [ { - "hashed_secret": "fe54e5e937d642307ec155b47ac8a214cb40d474", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 10, - "type": "Base64 High Entropy String" + "line_number": 40 }, { - "hashed_secret": "cea0e701e53c42bede2212b22f58f9ff8324da55", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 13, - "type": "Base64 High Entropy String" + "line_number": 43 }, { - "hashed_secret": "d98d72830f08c9a8b96ed11d3d96ae9e71b72a26", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 16, - "type": "Base64 High Entropy String" + "line_number": 46 }, { - "hashed_secret": "667fd45d415f73f4132cf0ed11452beb51117b12", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 18, - "type": "Base64 High Entropy String" + "line_number": 53 }, { - "hashed_secret": "c2599d515ba3be74ed58821485ba769fc565e424", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 33, - "type": "Base64 High Entropy String" + "line_number": 56 }, { - "hashed_secret": "6ec5eb29e2884f0c9731493b38902e37c2d672ba", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 35, - "type": "Base64 High Entropy String" + "line_number": 59 }, { - "hashed_secret": "99126b74731670a59b663d5320712564ec7b5f22", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-init-proxy-groups-job.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", "is_verified": false, - "line_number": 36, - "type": "Base64 High Entropy String" + "line_number": 62 } ], - "gen3/test/secretsTest.sh": [ + "kube/services/jobs/google-manage-account-access-cronjob.yaml": [ { - "hashed_secret": "c2c715092ef59cba22520f109f041efca84b8938", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 25, - "type": "Secret Keyword" - } - ], - "gen3/test/terraformTest.sh": [ + "line_number": 48 + }, { - "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 156, - "type": "Secret Keyword" + "line_number": 51 }, { - "hashed_secret": "1cc07dccfdf640eb0e403e490a873a5536759009", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 172, - "type": "Base64 High Entropy String" + "line_number": 54 }, { - "hashed_secret": "185a71a740ef6b9b21c84e6eaa47b89c7de181ef", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 61 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 175, - "type": "Base64 High Entropy String" + "line_number": 64 }, { - "hashed_secret": "329b7cd8191942bedd337107934d365c43a86e6c", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-cronjob.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 175, - "type": "Secret Keyword" + "line_number": 67 } ], - "kube/services/argocd/values.yaml": [ + "kube/services/jobs/google-manage-account-access-job.yaml": [ { - "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 360, - "type": "Private Key" + "line_number": 40 }, { - "hashed_secret": "edbd5e119f94badb9f99a67ac6ff4c7a5204ad61", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 379, - "type": "Secret Keyword" + "line_number": 43 }, { - "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 412, - "type": "Secret Keyword" - } - ], - "kube/services/datadog/values.yaml": [ + "line_number": 46 + }, { - "hashed_secret": "4a8ce7ae6a8a7f2624e232b61b18c2ac9789c44b", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", + "is_verified": false, + "line_number": 53 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", + "is_verified": false, + "line_number": 56 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-account-access-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 23, - "type": "Secret Keyword" + "line_number": 59 } ], - "kube/services/fenceshib/fenceshib-configmap.yaml": [ + "kube/services/jobs/google-manage-keys-cronjob.yaml": [ { - "hashed_secret": "a985e14b9d6744a2d04f29347693b55c116e478c", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 375, - "type": "Base64 High Entropy String" + "line_number": 48 }, { - "hashed_secret": "adc747bc5eb82ef4b017f5c3759dcee5aa28c36f", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 376, - "type": "Base64 High Entropy String" + "line_number": 51 }, { - "hashed_secret": "59b1702ff0eaf92c9271cbd12f587de97df7e13b", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 377, - "type": "Base64 High Entropy String" + "line_number": 54 }, { - "hashed_secret": "b4a748bbfbbca8925d932a47ab3dcb970d34caf5", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 378, - "type": "Base64 High Entropy String" + "line_number": 61 }, { - "hashed_secret": "af646701a84f7dd9f0e87753f54def881326e78a", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 379, - "type": "Base64 High Entropy String" + "line_number": 64 }, { - "hashed_secret": "20c15ad9742124dc06e1612282c49bb443ebcbd9", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-cronjob.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 380, - "type": "Base64 High Entropy String" - }, + "line_number": 67 + } + ], + "kube/services/jobs/google-manage-keys-job.yaml": [ { - "hashed_secret": "9caded71b967a11b7a6cd0f20db91f06f3517d12", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 381, - "type": "Base64 High Entropy String" + "line_number": 40 }, { - "hashed_secret": "8f19501bc9241b71f7b6db929fb35ab12635dcd7", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 382, - "type": "Base64 High Entropy String" + "line_number": 43 }, { - "hashed_secret": "d6220f6a55df1ed11c4250f42ab07bb9da20541a", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 383, - "type": "Base64 High Entropy String" + "line_number": 46 }, { - "hashed_secret": "dadd9b96636f9529f2547d05d754dc310ceba0c3", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 384, - "type": "Base64 High Entropy String" + "line_number": 53 }, { - "hashed_secret": "3074bc66584550e20c3697a28f67a0762394943c", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 385, - "type": "Base64 High Entropy String" + "line_number": 56 }, { - "hashed_secret": "823131319b4c4b4688f44d3e832bfa9696f16b52", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-manage-keys-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 386, - "type": "Base64 High Entropy String" - }, + "line_number": 59 + } + ], + "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml": [ { - "hashed_secret": "015b780cbfb76988caf52de8ac974a6781e53110", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 387, - "type": "Base64 High Entropy String" + "line_number": 48 }, { - "hashed_secret": "5c8fac33207d74d667680ade09447ea8f43b76d7", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 388, - "type": "Base64 High Entropy String" + "line_number": 51 }, { - "hashed_secret": "c0c4bb09d8394e8f001e337bd27ccac355433d9e", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 389, - "type": "Base64 High Entropy String" + "line_number": 54 }, { - "hashed_secret": "f95631bcbbbc56e18487dcb242cfb1b3e74b16a1", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 390, - "type": "Base64 High Entropy String" + "line_number": 61 }, { - "hashed_secret": "01a692ab6232e0882a313d148981bab58ab98f53", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 391, - "type": "Base64 High Entropy String" + "line_number": 64 }, { - "hashed_secret": "658060a680d415ce6690ad2c3b622ddb33ddd50a", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 392, - "type": "Base64 High Entropy String" - }, + "line_number": 67 + } + ], + "kube/services/jobs/google-verify-bucket-access-group-job.yaml": [ { - "hashed_secret": "80915b0bd9daa5e1f95cad573892980b1b5a2294", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 393, - "type": "Base64 High Entropy String" + "line_number": 40 }, { - "hashed_secret": "cc55977b293d8cdca8a2c19dfea6874e70057c41", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 394, - "type": "Base64 High Entropy String" + "line_number": 43 }, { - "hashed_secret": "e400ed02add75dd5f3a8c212857acf12027437d1", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 395, - "type": "Base64 High Entropy String" + "line_number": 46 }, { - "hashed_secret": "2e819c8baa3b0508a32b77de258655b3f3a6f7cb", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 396, - "type": "Base64 High Entropy String" + "line_number": 53 }, { - "hashed_secret": "546ed926d58ea5492ab6adb8be94a67aa44ac433", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 397, - "type": "Base64 High Entropy String" + "line_number": 56 }, { - "hashed_secret": "f056f2deceed268e7af6dbdaf2577079c76e006a", + "type": "Secret Keyword", + "filename": "kube/services/jobs/google-verify-bucket-access-group-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 398, - "type": "Base64 High Entropy String" - }, + "line_number": 59 + } + ], + "kube/services/jobs/graph-create-job.yaml": [ { - "hashed_secret": "d75efee28f4798c3a9c6f44b78a8500513ef28b2", + "type": "Secret Keyword", + "filename": "kube/services/jobs/graph-create-job.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 399, - "type": "Base64 High Entropy String" - }, + "line_number": 33 + } + ], + "kube/services/jobs/indexd-authz-job.yaml": [ { - "hashed_secret": "fbad0bc8f7792b03f89cd3780eb7cf79f284c525", + "type": "Secret Keyword", + "filename": "kube/services/jobs/indexd-authz-job.yaml", + "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3", "is_verified": false, - "line_number": 419, - "type": "Base64 High Entropy String" + "line_number": 32 }, { - "hashed_secret": "3f6480956a775dacb44e2c39aa3d4722a347f7ab", + "type": "Secret Keyword", + "filename": "kube/services/jobs/indexd-authz-job.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 420, - "type": "Base64 High Entropy String" + "line_number": 35 }, { - "hashed_secret": "17f32ae55b14d708ca121722c2cae37189f19daf", + "type": "Secret Keyword", + "filename": "kube/services/jobs/indexd-authz-job.yaml", + "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039", "is_verified": false, - "line_number": 423, - "type": "Base64 High Entropy String" - }, + "line_number": 38 + } + ], + "kube/services/jobs/indexd-userdb-job.yaml": [ { - "hashed_secret": "08a74689ca077515d406093720a7e5675fb42bb8", + "type": "Secret Keyword", + "filename": "kube/services/jobs/indexd-userdb-job.yaml", + "hashed_secret": "0b701c1fabb6ba47a7d47d455e3696d207014bd3", "is_verified": false, - "line_number": 424, - "type": "Base64 High Entropy String" + "line_number": 40 }, { - "hashed_secret": "fa577bb3b2600d2d522dcfea8f1e34896760fcf2", + "type": "Secret Keyword", + "filename": "kube/services/jobs/indexd-userdb-job.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 425, - "type": "Base64 High Entropy String" + "line_number": 43 }, { - "hashed_secret": "37254f15cca211a1bd5f7ceb23de2b3eb8fb33aa", + "type": "Secret Keyword", + "filename": "kube/services/jobs/indexd-userdb-job.yaml", + "hashed_secret": "aee98a99696237d70b6854ee4c2d9e42bc696039", "is_verified": false, - "line_number": 426, - "type": "Base64 High Entropy String" - }, + "line_number": 46 + } + ], + "kube/services/jobs/metadata-aggregate-sync-job.yaml": [ { - "hashed_secret": "86865593e038509467b91c2d5f36ccc09c3f422b", + "type": "Secret Keyword", + "filename": "kube/services/jobs/metadata-aggregate-sync-job.yaml", + "hashed_secret": "e14f65c8ca7f3b27a0f0f5463569954841e162c9", "is_verified": false, - "line_number": 427, - "type": "Base64 High Entropy String" + "line_number": 31 }, { - "hashed_secret": "a899a8d9e114b2a8e108f90e6a72c056db22489f", + "type": "Secret Keyword", + "filename": "kube/services/jobs/metadata-aggregate-sync-job.yaml", + "hashed_secret": "c27babf45eb0ed87329e69c7d47dba611e859c5d", "is_verified": false, - "line_number": 428, - "type": "Base64 High Entropy String" - }, + "line_number": 34 + } + ], + "kube/services/jobs/metadata-delete-expired-objects-job.yaml": [ { - "hashed_secret": "756b4825f886afd83c25563ac9d45f318d695c48", + "type": "Secret Keyword", + "filename": "kube/services/jobs/metadata-delete-expired-objects-job.yaml", + "hashed_secret": "0cc8bac3fabe63722716d1e6fe04a8dded1e3ad0", "is_verified": false, - "line_number": 429, - "type": "Base64 High Entropy String" - }, + "line_number": 24 + } + ], + "kube/services/jobs/remove-objects-from-clouds-job.yaml": [ { - "hashed_secret": "89882eeb0aca97717a7e4afcf4bc08d077813c7f", + "type": "Secret Keyword", + "filename": "kube/services/jobs/remove-objects-from-clouds-job.yaml", + "hashed_secret": "deb02468778f4041fb189654698ac948e436732d", "is_verified": false, - "line_number": 430, - "type": "Base64 High Entropy String" + "line_number": 34 }, { - "hashed_secret": "347140d7b7ceb4e501c3c9c2ea4f29338e2f145e", + "type": "Secret Keyword", + "filename": "kube/services/jobs/remove-objects-from-clouds-job.yaml", + "hashed_secret": "b6f0ec0b08da77656ced48427841e28d7a8a81d6", "is_verified": false, - "line_number": 431, - "type": "Base64 High Entropy String" + "line_number": 37 }, { - "hashed_secret": "61dbf70eb10d609e60c7b87faf8f755ff48abc46", + "type": "Secret Keyword", + "filename": "kube/services/jobs/remove-objects-from-clouds-job.yaml", + "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb", "is_verified": false, - "line_number": 432, - "type": "Base64 High Entropy String" - }, + "line_number": 43 + } + ], + "kube/services/jobs/replicate-validation-job.yaml": [ { - "hashed_secret": "24cd54c4b2f58378bba008cb2df68ac663fba7c8", + "type": "Secret Keyword", + "filename": "kube/services/jobs/replicate-validation-job.yaml", + "hashed_secret": "deb02468778f4041fb189654698ac948e436732d", "is_verified": false, - "line_number": 433, - "type": "Base64 High Entropy String" + "line_number": 34 }, { - "hashed_secret": "fa4f9626ae4b98f4b61203c5bafb6f21c9c31e5d", + "type": "Secret Keyword", + "filename": "kube/services/jobs/replicate-validation-job.yaml", + "hashed_secret": "b6f0ec0b08da77656ced48427841e28d7a8a81d6", "is_verified": false, - "line_number": 434, - "type": "Base64 High Entropy String" + "line_number": 37 }, { - "hashed_secret": "b1370003d9cc1e346c83dba33e0418c7775a0c15", + "type": "Secret Keyword", + "filename": "kube/services/jobs/replicate-validation-job.yaml", + "hashed_secret": "abe72fcb190ed9c73eb20e198c73a97605b95063", "is_verified": false, - "line_number": 435, - "type": "Base64 High Entropy String" + "line_number": 40 }, { - "hashed_secret": "c66526e195e423a7ba7d68ac661cdcd8600dcd1f", + "type": "Secret Keyword", + "filename": "kube/services/jobs/replicate-validation-job.yaml", + "hashed_secret": "ca3cdac59f2bfa45cb014190e4509bf6becf28fb", "is_verified": false, - "line_number": 436, - "type": "Base64 High Entropy String" - }, + "line_number": 43 + } + ], + "kube/services/jobs/s3sync-cronjob.yaml": [ { - "hashed_secret": "d29d7044f0944eb30e02cf445f6998e3343dd811", + "type": "Secret Keyword", + "filename": "kube/services/jobs/s3sync-cronjob.yaml", + "hashed_secret": "27f6dfe15698a3bfaa183c84701cfb2bf4115415", "is_verified": false, - "line_number": 437, - "type": "Base64 High Entropy String" - }, + "line_number": 44 + } + ], + "kube/services/jobs/usersync-job.yaml": [ { - "hashed_secret": "80a869460f33722387d8d58e7d9d2e1bbd5d1fe1", + "type": "Secret Keyword", + "filename": "kube/services/jobs/usersync-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 438, - "type": "Base64 High Entropy String" + "line_number": 64 }, { - "hashed_secret": "4a06e2a02cbc665adccb4162dc57836895da65b8", + "type": "Secret Keyword", + "filename": "kube/services/jobs/usersync-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 439, - "type": "Base64 High Entropy String" + "line_number": 67 }, { - "hashed_secret": "ba2549f35835dfa101d3f660f7604dc78e3e226f", + "type": "Secret Keyword", + "filename": "kube/services/jobs/usersync-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 440, - "type": "Base64 High Entropy String" + "line_number": 70 }, { - "hashed_secret": "f354d4ee5fdb94ad29c7b3600264467f45b80eaa", + "type": "Secret Keyword", + "filename": "kube/services/jobs/usersync-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 441, - "type": "Base64 High Entropy String" + "line_number": 77 }, { - "hashed_secret": "bf17b587868ba7c3db9865b114261b5b8f1df870", + "type": "Secret Keyword", + "filename": "kube/services/jobs/usersync-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 442, - "type": "Base64 High Entropy String" + "line_number": 80 }, { - "hashed_secret": "de1fd7a0d32cba528b4d80818c6601f2588d5383", + "type": "Secret Keyword", + "filename": "kube/services/jobs/usersync-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 443, - "type": "Base64 High Entropy String" + "line_number": 83 }, { - "hashed_secret": "bcad65055f6de654541db2bf27d4e27bd54d94c7", + "type": "Secret Keyword", + "filename": "kube/services/jobs/usersync-job.yaml", + "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73", "is_verified": false, - "line_number": 444, - "type": "Base64 High Entropy String" - }, + "line_number": 86 + } + ], + "kube/services/jobs/useryaml-job.yaml": [ { - "hashed_secret": "f2e16f2dd532f65f79341342fdf57a093fc408d8", + "type": "Secret Keyword", + "filename": "kube/services/jobs/useryaml-job.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 445, - "type": "Base64 High Entropy String" + "line_number": 40 }, { - "hashed_secret": "bb036a679a7d2df9fd2ca57068a446bf7f7dd106", + "type": "Secret Keyword", + "filename": "kube/services/jobs/useryaml-job.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 446, - "type": "Base64 High Entropy String" + "line_number": 43 }, { - "hashed_secret": "5aa6568b1e8185578a6e964f5c322783ad349554", + "type": "Secret Keyword", + "filename": "kube/services/jobs/useryaml-job.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 447, - "type": "Base64 High Entropy String" + "line_number": 46 }, { - "hashed_secret": "4d14835ff0b0bf5aad480296cb705c74ac65f413", + "type": "Secret Keyword", + "filename": "kube/services/jobs/useryaml-job.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 448, - "type": "Base64 High Entropy String" + "line_number": 53 }, { - "hashed_secret": "3f23f77dcf454ad73c4d61c44fd9aa584ef946c1", + "type": "Secret Keyword", + "filename": "kube/services/jobs/useryaml-job.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 451, - "type": "Base64 High Entropy String" + "line_number": 56 }, { - "hashed_secret": "1739fe5e5dfcf851b64f8b7b11538f1de29ce0b5", + "type": "Secret Keyword", + "filename": "kube/services/jobs/useryaml-job.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 452, - "type": "Base64 High Entropy String" + "line_number": 59 }, { - "hashed_secret": "8129db302110714fc735e3494bd82a65690e0963", + "type": "Secret Keyword", + "filename": "kube/services/jobs/useryaml-job.yaml", + "hashed_secret": "ea73fcfdaa415890d5fde24d3b2245671be32f73", + "is_verified": false, + "line_number": 65 + } + ], + "kube/services/kayako-wrapper/kayako-wrapper-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/kayako-wrapper/kayako-wrapper-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 453, - "type": "Base64 High Entropy String" + "line_number": 56 }, { - "hashed_secret": "b48bfc62091164086a703115a0e68bdb09212591", + "type": "Secret Keyword", + "filename": "kube/services/kayako-wrapper/kayako-wrapper-deploy.yaml", + "hashed_secret": "fb7ea689a364feb7aafbf8d553eb77073fa7ba11", "is_verified": false, - "line_number": 454, - "type": "Base64 High Entropy String" + "line_number": 59 + } + ], + "kube/services/kubecost-standalone/thanos-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/kubecost-standalone/thanos-deploy.yaml", + "hashed_secret": "064376809efc3acda5bd341aca977e149b989696", + "is_verified": false, + "line_number": 127 + } + ], + "kube/services/kubecost-standalone/values.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/kubecost-standalone/values.yaml", + "hashed_secret": "ec9786daee68e3541963a51299160859fe4db663", + "is_verified": false, + "line_number": 30 + } + ], + "kube/services/manifestservice/manifestservice-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/manifestservice/manifestservice-deploy.yaml", + "hashed_secret": "3da2c49c267b6c58401bbf05e379b38d20434f78", + "is_verified": false, + "line_number": 61 }, { - "hashed_secret": "a10284feaf27f84081073a3267e3dce24ca7b911", + "type": "Secret Keyword", + "filename": "kube/services/manifestservice/manifestservice-deploy.yaml", + "hashed_secret": "469e0c2b1a67aa94955bae023ddc727be31581a7", "is_verified": false, - "line_number": 455, - "type": "Base64 High Entropy String" + "line_number": 64 }, { - "hashed_secret": "3fd80f31de4be8dde9d2b421e832c7d4043fd49a", + "type": "Secret Keyword", + "filename": "kube/services/manifestservice/manifestservice-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 456, - "type": "Base64 High Entropy String" + "line_number": 67 } ], - "kube/services/jobs/indexd-authz-job.yaml": [ + "kube/services/metadata/metadata-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/metadata/metadata-deploy.yaml", + "hashed_secret": "e14f65c8ca7f3b27a0f0f5463569954841e162c9", + "is_verified": false, + "line_number": 61 + }, { - "hashed_secret": "bf21a9e8fbc5a3846fb05b4fa0859e0917b2202f", + "type": "Secret Keyword", + "filename": "kube/services/metadata/metadata-deploy.yaml", + "hashed_secret": "c27babf45eb0ed87329e69c7d47dba611e859c5d", "is_verified": false, - "line_number": 87, - "type": "Basic Auth Credentials" + "line_number": 66 } ], "kube/services/monitoring/grafana-values.yaml": [ { + "type": "Secret Keyword", + "filename": "kube/services/monitoring/grafana-values.yaml", "hashed_secret": "2ae868079d293e0a185c671c7bcdac51df36e385", "is_verified": false, - "line_number": 162, - "type": "Secret Keyword" + "line_number": 162 }, { - "hashed_secret": "7a64ff8446b06d38dc271019994f13823a2cbcf4", + "type": "Secret Keyword", + "filename": "kube/services/monitoring/grafana-values.yaml", + "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 166, - "type": "Secret Keyword" + "line_number": 331 } ], - "kube/services/revproxy/helpers.js": [ + "kube/services/monitoring/thanos-deploy.yaml": [ { - "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af", + "type": "Secret Keyword", + "filename": "kube/services/monitoring/thanos-deploy.yaml", + "hashed_secret": "064376809efc3acda5bd341aca977e149b989696", "is_verified": false, - "line_number": 10, - "type": "Base64 High Entropy String" + "line_number": 130 } ], - "kube/services/revproxy/helpersTest.js": [ + "kube/services/ohif-viewer/ohif-viewer-deploy.yaml": [ { - "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb", + "type": "Secret Keyword", + "filename": "kube/services/ohif-viewer/ohif-viewer-deploy.yaml", + "hashed_secret": "3f87db80519a9ae7d8112f4e0d4cc81441181818", "is_verified": false, - "line_number": 22, - "type": "JSON Web Token" + "line_number": 40 } ], - "kube/services/superset/superset-deploy.yaml": [ + "kube/services/orthanc/orthanc-deploy.yaml": [ { - "hashed_secret": "96e4aceb7cf284be363aa248a32a7cc89785a9f7", + "type": "Secret Keyword", + "filename": "kube/services/orthanc/orthanc-deploy.yaml", + "hashed_secret": "3f87db80519a9ae7d8112f4e0d4cc81441181818", "is_verified": false, - "line_number": 38, - "type": "Secret Keyword" + "line_number": 41 } ], - "kube/services/superset/superset-redis.yaml": [ + "kube/services/peregrine/peregrine-canary-deploy.yaml": [ { - "hashed_secret": "4af3596275edcb7cd5cc6c3c38bc10479902a08f", + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml", + "hashed_secret": "6131c35d7eebdbc17a314bef8aac75b87323cff3", "is_verified": false, - "line_number": 165, - "type": "Secret Keyword" + "line_number": 61 }, { - "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9", + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml", + "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966", + "is_verified": false, + "line_number": 64 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml", + "hashed_secret": "990a3202b5c94aa5e5997e7dc1a218e457f8b8ec", + "is_verified": false, + "line_number": 70 + }, + { + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-canary-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 265, - "type": "Secret Keyword" + "line_number": 73 } ], - "kube/services/superset/values.yaml": [ + "kube/services/peregrine/peregrine-deploy.yaml": [ { - "hashed_secret": "6f803b24314c39062efe38d0c1da8c472f47eab3", + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-deploy.yaml", + "hashed_secret": "6131c35d7eebdbc17a314bef8aac75b87323cff3", "is_verified": false, - "line_number": 54, - "type": "Secret Keyword" + "line_number": 67 }, { - "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46", + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-deploy.yaml", + "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966", "is_verified": false, - "line_number": 86, - "type": "Secret Keyword" + "line_number": 70 }, { - "hashed_secret": "3eb416223e9e69e6bb8ee19793911ad1ad2027d8", + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-deploy.yaml", + "hashed_secret": "990a3202b5c94aa5e5997e7dc1a218e457f8b8ec", "is_verified": false, - "line_number": 212, - "type": "Secret Keyword" + "line_number": 76 }, { - "hashed_secret": "ff55435345834a3fe224936776c2aa15f6ed5358", + "type": "Secret Keyword", + "filename": "kube/services/peregrine/peregrine-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 79 + } + ], + "kube/services/pidgin/pidgin-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/pidgin/pidgin-deploy.yaml", + "hashed_secret": "49af232c7adfcd54a40202e06261396a757e4ddd", "is_verified": false, - "line_number": 396, - "type": "Secret Keyword" + "line_number": 59 }, { - "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc", + "type": "Secret Keyword", + "filename": "kube/services/pidgin/pidgin-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 503, - "type": "Secret Keyword" + "line_number": 62 } ], - "package-lock.json": [ + "kube/services/portal/portal-deploy.yaml": [ { - "hashed_secret": "0656ad0df3af4633dc369f13d5e8806973c5fd9d", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 1481, - "type": "Base64 High Entropy String" + "line_number": 55 }, { - "hashed_secret": "00091d875d922437c5fc9e6067a08e78c2482e87", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-deploy.yaml", + "hashed_secret": "5c5a8e158ad2d8544f73cd5422072d414f497faa", "is_verified": false, - "line_number": 1489, - "type": "Base64 High Entropy String" + "line_number": 58 }, { - "hashed_secret": "c4e5cc37e115bf7d86e76e3d799705bf691e4d00", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-deploy.yaml", + "hashed_secret": "619551216e129bbc5322678abf9c9210c0327cfb", "is_verified": false, - "line_number": 1521, - "type": "Base64 High Entropy String" + "line_number": 61 }, { - "hashed_secret": "0512e37fbedf1d16828680a038a241b4780a5c04", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-deploy.yaml", + "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9", "is_verified": false, - "line_number": 1547, - "type": "Base64 High Entropy String" - }, + "line_number": 67 + } + ], + "kube/services/portal/portal-root-deploy.yaml": [ { - "hashed_secret": "01868fd50edbfe6eb91e5b01209b543adc6857af", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-root-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 1611, - "type": "Base64 High Entropy String" + "line_number": 55 }, { - "hashed_secret": "a6f48bf1e398deffc7fd31da17c3506b46c97a93", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-root-deploy.yaml", + "hashed_secret": "5c5a8e158ad2d8544f73cd5422072d414f497faa", "is_verified": false, - "line_number": 1640, - "type": "Base64 High Entropy String" + "line_number": 58 }, { - "hashed_secret": "85ce358dbdec0996cf3ccd2bf1c6602af68c181e", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-root-deploy.yaml", + "hashed_secret": "619551216e129bbc5322678abf9c9210c0327cfb", "is_verified": false, - "line_number": 1648, - "type": "Base64 High Entropy String" + "line_number": 61 }, { - "hashed_secret": "6f9bfb49cb818d2fe07592515e4c3f7a0bbd7e0e", + "type": "Secret Keyword", + "filename": "kube/services/portal/portal-root-deploy.yaml", + "hashed_secret": "e3c7565314f404e3883929f003c65a02a80366e9", "is_verified": false, - "line_number": 1664, - "type": "Base64 High Entropy String" - }, + "line_number": 67 + } + ], + "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml": [ { - "hashed_secret": "7098a3e6d6d2ec0a40f04fe12509c5c6f4c49c0e", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "dbd5f43594a152b52261c8e21520a3989823fe55", "is_verified": false, - "line_number": 1683, - "type": "Base64 High Entropy String" + "line_number": 74 }, { - "hashed_secret": "1664ad175bba1795a7ecad572bae7e0740b94f56", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "1c062eaac9e6fa0766377d3cfc3e4a88982fecdb", "is_verified": false, - "line_number": 1733, - "type": "Base64 High Entropy String" + "line_number": 77 }, { - "hashed_secret": "1ec4ce2eb945ce2f816dcb6ebdd1e10247f439a3", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "694cfd0a009a42055e975de9111b2f3c6e8a3634", "is_verified": false, - "line_number": 1742, - "type": "Base64 High Entropy String" + "line_number": 80 }, { - "hashed_secret": "a7af5768a6d936e36f28e1030d7f894d7aaf555e", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "4b09a441cef18c75560f6c3caeafc96f2163c3fd", "is_verified": false, - "line_number": 1755, - "type": "Base64 High Entropy String" + "line_number": 90 }, { - "hashed_secret": "6fbc7dd864586173160874f2a86ca7d2d552cb85", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "7e7478a28dcc3695a083b66b47243b050c813e2d", "is_verified": false, - "line_number": 1769, - "type": "Base64 High Entropy String" + "line_number": 93 }, { - "hashed_secret": "81a961f2c89c6209328b74a8768e30fd76c3ac72", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "2f57bb00fcb93481c2be444e3e9f322b6cb5fadb", "is_verified": false, - "line_number": 1855, - "type": "Base64 High Entropy String" + "line_number": 96 }, { - "hashed_secret": "797d4751c536c421cb82b9f62e0a804af30d78f5", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "98f5a68541a6d981bf5825f23dffe6a0b150e457", "is_verified": false, - "line_number": 1889, - "type": "Base64 High Entropy String" + "line_number": 99 }, { - "hashed_secret": "0d55babfa89f240142c0adfc7b560500a1d3ae7c", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "0849046cdafcdb17f5a4bf5c528430d5e04ad295", "is_verified": false, - "line_number": 1894, - "type": "Base64 High Entropy String" + "line_number": 102 }, { - "hashed_secret": "e9fdc3025cd10bd8aa4508611e6b7b7a9d650a2c", + "type": "Secret Keyword", + "filename": "kube/services/presigned-url-fence/presigned-url-fence-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 1921, - "type": "Base64 High Entropy String" - }, + "line_number": 105 + } + ], + "kube/services/qa-dashboard/qa-dashboard-deployment.yaml": [ { - "hashed_secret": "4cf9419259c0ce8eee84b468af3c72db8b001620", + "type": "Secret Keyword", + "filename": "kube/services/qa-dashboard/qa-dashboard-deployment.yaml", + "hashed_secret": "253939a955a575ac69f409e5914dd0191b704760", "is_verified": false, - "line_number": 1950, - "type": "Base64 High Entropy String" - }, + "line_number": 63 + } + ], + "kube/services/qabot/qabot-deploy.yaml": [ { - "hashed_secret": "24816e3eb4308e247bde7c1d09ffb7b79c519b71", + "type": "Secret Keyword", + "filename": "kube/services/qabot/qabot-deploy.yaml", + "hashed_secret": "a9fa7aa8c08b647c3fb696e6598642d4a63e25be", "is_verified": false, - "line_number": 1983, - "type": "Base64 High Entropy String" - }, + "line_number": 86 + } + ], + "kube/services/requestor/requestor-deploy.yaml": [ { - "hashed_secret": "e9adfe8a333d45f4776fe0eab31608be5d7b6a7d", + "type": "Secret Keyword", + "filename": "kube/services/requestor/requestor-deploy.yaml", + "hashed_secret": "15debe4170aa5b89858d939f4c0644307ae7789b", "is_verified": false, - "line_number": 2004, - "type": "Base64 High Entropy String" - }, + "line_number": 61 + } + ], + "kube/services/revproxy/gen3.nginx.conf/indexd-service.conf": [ { - "hashed_secret": "03d6fb388dd1b185129b14221f7127715822ece6", + "type": "Secret Keyword", + "filename": "kube/services/revproxy/gen3.nginx.conf/indexd-service.conf", + "hashed_secret": "f89523833036f85fed37ce3ebf25492189bc9397", "is_verified": false, - "line_number": 2013, - "type": "Base64 High Entropy String" - }, + "line_number": 41 + } + ], + "kube/services/revproxy/gen3.nginx.conf/metadata-service.conf": [ { - "hashed_secret": "ee161bb3f899720f95cee50a5f9ef9c9ed96278b", + "type": "Secret Keyword", + "filename": "kube/services/revproxy/gen3.nginx.conf/metadata-service.conf", + "hashed_secret": "18c0871af26eb9875c0f840b13211f097c133fd2", "is_verified": false, - "line_number": 2046, - "type": "Base64 High Entropy String" - }, + "line_number": 24 + } + ], + "kube/services/revproxy/helpers.js": [ { - "hashed_secret": "ebeb5b574fa1ed24a40248275e6136759e766466", + "type": "Base64 High Entropy String", + "filename": "kube/services/revproxy/helpers.js", + "hashed_secret": "1d278d3c888d1a2fa7eed622bfc02927ce4049af", "is_verified": false, - "line_number": 2078, - "type": "Base64 High Entropy String" - }, + "line_number": 10 + } + ], + "kube/services/revproxy/helpersTest.js": [ { - "hashed_secret": "a6a555a428522ccf439fd516ce7c7e269274363f", + "type": "Base64 High Entropy String", + "filename": "kube/services/revproxy/helpersTest.js", + "hashed_secret": "389c3ec21b7325359051e97ff569b078843d2d37", "is_verified": false, - "line_number": 2083, - "type": "Base64 High Entropy String" + "line_number": 19 }, { - "hashed_secret": "f7f85d9f7c87f1e576dcaf4cf50f35728f9a3265", + "type": "JSON Web Token", + "filename": "kube/services/revproxy/helpersTest.js", + "hashed_secret": "e029d4904cc728879d70030572bf37d4510367cb", "is_verified": false, - "line_number": 2111, - "type": "Base64 High Entropy String" - }, + "line_number": 22 + } + ], + "kube/services/revproxy/revproxy-deploy.yaml": [ { - "hashed_secret": "3f1646b60abe74297d2f37a1eee5dc771ad834fc", + "type": "Secret Keyword", + "filename": "kube/services/revproxy/revproxy-deploy.yaml", + "hashed_secret": "c7a87a61893a647e29289845cb51e61afb06800b", "is_verified": false, - "line_number": 2138, - "type": "Base64 High Entropy String" + "line_number": 74 }, { - "hashed_secret": "fd933c71e82d5519ae0cb0779b370d02f6935759", + "type": "Secret Keyword", + "filename": "kube/services/revproxy/revproxy-deploy.yaml", + "hashed_secret": "b3a4e2dea4c1fae8c58a07a84065b73b3a2d831c", "is_verified": false, - "line_number": 2143, - "type": "Base64 High Entropy String" + "line_number": 77 }, { - "hashed_secret": "7090aa59cb52ad1f1810b08c4ac1ddf5c8fce523", + "type": "Secret Keyword", + "filename": "kube/services/revproxy/revproxy-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", "is_verified": false, - "line_number": 2150, - "type": "Base64 High Entropy String" - }, + "line_number": 80 + } + ], + "kube/services/sftp/sftp-deploy.yaml": [ { - "hashed_secret": "756444bea4ea3d67844d8ddf58ad32356e9c2430", + "type": "Secret Keyword", + "filename": "kube/services/sftp/sftp-deploy.yaml", + "hashed_secret": "9fdebf62e477d59d25730744c8b3089c67c3db85", "is_verified": false, - "line_number": 2188, - "type": "Base64 High Entropy String" - }, + "line_number": 39 + } + ], + "kube/services/sheepdog/sheepdog-canary-deploy.yaml": [ { - "hashed_secret": "f74135fdd6b8dafdfb01ebbc61c5e5c24ee27cf8", + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml", + "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d", "is_verified": false, - "line_number": 2291, - "type": "Base64 High Entropy String" + "line_number": 58 }, { - "hashed_secret": "56fbae787f4aed7d0632e95840d71bd378d3a36f", + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 2303, - "type": "Base64 High Entropy String" + "line_number": 61 }, { - "hashed_secret": "81cb6be182eb79444202c4563080aee75296a672", + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml", + "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2", "is_verified": false, - "line_number": 2308, - "type": "Base64 High Entropy String" + "line_number": 67 }, { - "hashed_secret": "f0f3f7bce32184893046ac5f8cc80da56c3ca539", + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-canary-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 70 + } + ], + "kube/services/sheepdog/sheepdog-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-deploy.yaml", + "hashed_secret": "ec9c944c51e87322de8d22e3ca9e2be1ad8fee0d", "is_verified": false, - "line_number": 2317, - "type": "Base64 High Entropy String" + "line_number": 63 }, { - "hashed_secret": "097893233346336f4003acfb6eb173ee59e648f0", + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-deploy.yaml", + "hashed_secret": "79496491225eda4a7be9fcddee2825c85b1535cc", "is_verified": false, - "line_number": 2327, - "type": "Base64 High Entropy String" + "line_number": 66 }, { - "hashed_secret": "bb14c3b4ef4a9f2e86ffdd44b88d9b6729419671", + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-deploy.yaml", + "hashed_secret": "e43756046ad1763d6946575fed0e05130a154bd2", "is_verified": false, - "line_number": 2332, - "type": "Base64 High Entropy String" + "line_number": 72 }, { - "hashed_secret": "71344a35cff67ef081920095d1406601fb5e9b97", + "type": "Secret Keyword", + "filename": "kube/services/sheepdog/sheepdog-deploy.yaml", + "hashed_secret": "9ce05cf6168d15dfe02aac9ca9e0712c19c9436d", + "is_verified": false, + "line_number": 75 + } + ], + "kube/services/shiny/shiny-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/shiny/shiny-deploy.yaml", + "hashed_secret": "327a1bbc6dc0ce857472ee9162a3415133862d50", + "is_verified": false, + "line_number": 43 + } + ], + "kube/services/ssjdispatcher/ssjdispatcher-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/ssjdispatcher/ssjdispatcher-deploy.yaml", + "hashed_secret": "7f932449df74fc78573fea502df8a484aef3f69d", + "is_verified": false, + "line_number": 61 + } + ], + "kube/services/superset/superset-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/superset/superset-deploy.yaml", + "hashed_secret": "3e9d1737117ff62b23e37aedc72b522b0134997a", "is_verified": false, - "line_number": 2340, - "type": "Base64 High Entropy String" + "line_number": 235 }, { - "hashed_secret": "eb3db6990fd43477a35dfeffc90b3f1ffa83c7bd", + "type": "Secret Keyword", + "filename": "kube/services/superset/superset-deploy.yaml", + "hashed_secret": "6ac08eaa58d425783ff8b5a38fe16ee66c0bce15", + "is_verified": false, + "line_number": 311 + } + ], + "kube/services/superset/superset-redis.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/superset/superset-redis.yaml", + "hashed_secret": "9fe1c31809da38c55b2b64bfab47b92bc5f6b7b9", + "is_verified": false, + "line_number": 265 + } + ], + "kube/services/superset/values.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/superset/values.yaml", + "hashed_secret": "9a09d4081ddc128a80384712ce6df3578e6bc58e", "is_verified": false, - "line_number": 2349, - "type": "Base64 High Entropy String" + "line_number": 173 }, { - "hashed_secret": "266288bdc14807b538d1e48a5891e361fa9b4a14", + "type": "Secret Keyword", + "filename": "kube/services/superset/values.yaml", + "hashed_secret": "118c413f3fc929a1624f4c3e1da1e3d24377a693", "is_verified": false, - "line_number": 2357, - "type": "Base64 High Entropy String" + "line_number": 299 }, { - "hashed_secret": "800477261175fd21f23e7321923e1fba6ae55471", + "type": "Secret Keyword", + "filename": "kube/services/superset/values.yaml", + "hashed_secret": "d2a8d1ddfa75398366cff06545380c73481ec17d", "is_verified": false, - "line_number": 2369, - "type": "Base64 High Entropy String" + "line_number": 445 }, { - "hashed_secret": "3f0c251b9c2c21454445a98fde6915ceacde2136", + "type": "Secret Keyword", + "filename": "kube/services/superset/values.yaml", + "hashed_secret": "98a84a63e5633d17e3b27b69695f87aa7189e9dc", + "is_verified": false, + "line_number": 459 + } + ], + "kube/services/thor/thor-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/thor/thor-deploy.yaml", + "hashed_secret": "1f3f96a3887209d0dda357e5516231ee9c5cd9a7", + "is_verified": false, + "line_number": 100 + } + ], + "kube/services/tube/tube-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/tube/tube-deploy.yaml", + "hashed_secret": "ca253d1c9dece2da0d6fb24ded7bdb849a475966", "is_verified": false, - "line_number": 2387, - "type": "Base64 High Entropy String" + "line_number": 58 } ], - "tf_files/aws/cognito/README.md": [ + "kube/services/ws-storage/ws-storage-deploy.yaml": [ { - "hashed_secret": "f6920f370a30262b7dd70e97293c73ec89739b70", + "type": "Secret Keyword", + "filename": "kube/services/ws-storage/ws-storage-deploy.yaml", + "hashed_secret": "ec2d9395e11f353370a4abac21a1565641b35ce9", "is_verified": false, - "line_number": 106, - "type": "Secret Keyword" + "line_number": 66 + } + ], + "kube/services/wts/wts-deploy.yaml": [ + { + "type": "Secret Keyword", + "filename": "kube/services/wts/wts-deploy.yaml", + "hashed_secret": "5de687ae886f19c3cb68d4980e3f2e77cca3db9e", + "is_verified": false, + "line_number": 65 + } + ], + "packer/buildAll.sh": [ + { + "type": "Secret Keyword", + "filename": "packer/buildAll.sh", + "hashed_secret": "6e1d66a1596528c308e601c10aa0b92d53606ab9", + "is_verified": false, + "line_number": 15 + } + ], + "packer/variables.example.json": [ + { + "type": "Secret Keyword", + "filename": "packer/variables.example.json", + "hashed_secret": "a3a0648a036bebf78ba1a1eb498a66081059da10", + "is_verified": false, + "line_number": 5 } ], "tf_files/aws/commons/README.md": [ { - "hashed_secret": "d02e53411e8cb4cd709778f173f7bc9a3455f8ed", + "type": "Secret Keyword", + "filename": "tf_files/aws/commons/README.md", + "hashed_secret": "5f02a3fb14ab1ce5c18c362b04b8ffc603ea5951", "is_verified": false, - "line_number": 60, - "type": "Secret Keyword" + "line_number": 60 }, { - "hashed_secret": "9dc0da3613af850c5a018b0a88a5626fb8888e4e", + "type": "Secret Keyword", + "filename": "tf_files/aws/commons/README.md", + "hashed_secret": "49cfceed8aa8df159e53aa5c5951cad48a3f1216", "is_verified": false, - "line_number": 78, - "type": "Secret Keyword" + "line_number": 67 + }, + { + "type": "Secret Keyword", + "filename": "tf_files/aws/commons/README.md", + "hashed_secret": "18ad13589ca5fb3c432d7d9f0fe49f8ed6e2c478", + "is_verified": false, + "line_number": 70 } ], "tf_files/aws/eks/sample.tfvars": [ { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/eks/sample.tfvars", "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", "is_verified": false, - "line_number": 107, - "type": "Hex High Entropy String" + "line_number": 107 } ], "tf_files/aws/eks/variables.tf": [ { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/eks/variables.tf", "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", "is_verified": false, - "line_number": 133, - "type": "Hex High Entropy String" + "line_number": 133 } ], "tf_files/aws/modules/common-logging/README.md": [ { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/common-logging/README.md", "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", "is_verified": false, - "line_number": 57, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", - "is_verified": false, - "line_number": 59, - "type": "Hex High Entropy String" + "line_number": 57 } ], "tf_files/aws/modules/common-logging/lambda_function.py": [ { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/common-logging/lambda_function.py", "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", "is_verified": false, - "line_number": 18, - "type": "Hex High Entropy String" + "line_number": 18 }, { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/common-logging/lambda_function.py", "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", "is_verified": false, - "line_number": 18, - "type": "Base64 High Entropy String" + "line_number": 18 + }, + { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/common-logging/lambda_function.py", + "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94", + "is_verified": false, + "line_number": 18 }, { - "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/common-logging/lambda_function.py", + "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a", "is_verified": false, - "line_number": 18, - "type": "Hex High Entropy String" + "line_number": 18 }, { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/common-logging/lambda_function.py", "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", "is_verified": false, - "line_number": 30, - "type": "Hex High Entropy String" + "line_number": 30 } ], "tf_files/aws/modules/common-logging/testLambda.py": [ { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/common-logging/testLambda.py", "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", "is_verified": false, - "line_number": 5, - "type": "Hex High Entropy String" + "line_number": 5 }, { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/common-logging/testLambda.py", "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", "is_verified": false, - "line_number": 5, - "type": "Base64 High Entropy String" + "line_number": 5 + }, + { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/common-logging/testLambda.py", + "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94", + "is_verified": false, + "line_number": 5 }, { - "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/common-logging/testLambda.py", + "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a", "is_verified": false, - "line_number": 5, - "type": "Hex High Entropy String" + "line_number": 5 + }, + { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/common-logging/testLambda.py", + "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", + "is_verified": false, + "line_number": 10 } ], "tf_files/aws/modules/eks/variables.tf": [ { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/eks/variables.tf", "hashed_secret": "83c1003f406f34fba4d6279a948fee3abc802884", "is_verified": false, - "line_number": 113, - "type": "Hex High Entropy String" + "line_number": 113 } ], "tf_files/aws/modules/management-logs/README.md": [ { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/management-logs/README.md", "hashed_secret": "83442aa5a16cb1992731c32367ef464564388017", "is_verified": false, - "line_number": 54, - "type": "Base64 High Entropy String" - }, - { - "hashed_secret": "fd4a4637ac99de2c1d89155d66d1f3de15d231a2", - "is_verified": false, - "line_number": 56, - "type": "Hex High Entropy String" + "line_number": 54 } ], "tf_files/aws/modules/management-logs/lambda_function.py": [ { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/management-logs/lambda_function.py", "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", "is_verified": false, - "line_number": 18, - "type": "Hex High Entropy String" + "line_number": 18 }, { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/management-logs/lambda_function.py", "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", "is_verified": false, - "line_number": 18, - "type": "Base64 High Entropy String" + "line_number": 18 + }, + { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/management-logs/lambda_function.py", + "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94", + "is_verified": false, + "line_number": 18 }, { - "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/management-logs/lambda_function.py", + "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a", "is_verified": false, - "line_number": 18, - "type": "Hex High Entropy String" + "line_number": 18 }, { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/management-logs/lambda_function.py", "hashed_secret": "4f9fd96d3926f2c53ab0261d33f1d1a85a6a77ff", "is_verified": false, - "line_number": 30, - "type": "Hex High Entropy String" + "line_number": 30 } ], "tf_files/aws/modules/management-logs/testLambda.py": [ { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/management-logs/testLambda.py", "hashed_secret": "061765d6854d72f03a6527610d5b6822c9d516de", "is_verified": false, - "line_number": 5, - "type": "Hex High Entropy String" + "line_number": 5 }, { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/management-logs/testLambda.py", "hashed_secret": "61df81a188bb4dba6ae6128ff7e2c9c6a6f736ef", "is_verified": false, - "line_number": 5, - "type": "Base64 High Entropy String" + "line_number": 5 }, { - "hashed_secret": "a4667450661f32f7ad0f06e2f893a8fee9f18e38", + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/management-logs/testLambda.py", + "hashed_secret": "a4752db26b4774d3429878f36ceb7b61805ffd94", "is_verified": false, - "line_number": 5, - "type": "Hex High Entropy String" + "line_number": 5 }, { - "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a", + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/management-logs/testLambda.py", + "hashed_secret": "b979d8d0c0e8413c20a5597f789e31f0a2b2ff3a", "is_verified": false, - "line_number": 6, - "type": "Base64 High Entropy String" + "line_number": 5 }, { - "hashed_secret": "51118900cd675df1b44f254057398f3e52902a5d", + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/management-logs/testLambda.py", + "hashed_secret": "3cf8eb4e9254e1d6cc523da01f8b798b9a83101a", "is_verified": false, - "line_number": 6, - "type": "Hex High Entropy String" + "line_number": 6 }, { + "type": "Hex High Entropy String", + "filename": "tf_files/aws/modules/management-logs/testLambda.py", "hashed_secret": "60a6dfc8d43cd2f5c6292899fc2f94f2d4fc32c4", "is_verified": false, - "line_number": 6, - "type": "Hex High Entropy String" + "line_number": 6 + }, + { + "type": "Base64 High Entropy String", + "filename": "tf_files/aws/modules/management-logs/testLambda.py", + "hashed_secret": "d484ccb4ced21e0149078377f14b913bf5c613d0", + "is_verified": false, + "line_number": 6 } ], "tf_files/aws/slurm/README.md": [ { - "hashed_secret": "fd85d792fa56981cf6a8d2a5c0857c74af86e99d", + "type": "Secret Keyword", + "filename": "tf_files/aws/slurm/README.md", + "hashed_secret": "c16686250cd583de64e02a47a8b194cd5578b2a1", "is_verified": false, - "line_number": 83, - "type": "Secret Keyword" + "line_number": 83 } ], "tf_files/azure/cloud.tf": [ { - "hashed_secret": "7c1a4b52b64e4106041971c345a1f3eab58fb2a4", + "type": "Secret Keyword", + "filename": "tf_files/azure/cloud.tf", + "hashed_secret": "38d930120a56321ceaa147b2bc1f19db53a0b993", "is_verified": false, - "line_number": 424, - "type": "Secret Keyword" + "line_number": 361 } ], "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars": [ { + "type": "Secret Keyword", + "filename": "tf_files/gcp-bwg/roots/commons_setup/variables/answerfile-commons_setup-001.template.tfvars", "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", "is_verified": false, - "line_number": 231, - "type": "Secret Keyword" + "line_number": 231 } ], "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars": [ { + "type": "Secret Keyword", + "filename": "tf_files/gcp-bwg/roots/templates/answerfile-commons_setup-001.template.tfvars", "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", "is_verified": false, - "line_number": 231, - "type": "Secret Keyword" + "line_number": 231 } ], "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP": [ { + "type": "Secret Keyword", + "filename": "tf_files/gcp-bwg/roots/templates/answerfile-env-tenant.user.tfvars_NO_APP_SETUP", "hashed_secret": "f865b53623b121fd34ee5426c792e5c33af8c227", "is_verified": false, - "line_number": 262, - "type": "Secret Keyword" + "line_number": 262 } ], - "tf_files/gcp/commons/sample.tfvars": [ + "tf_files/gcp/commons/root.tf": [ { - "hashed_secret": "9f29ed52bc91ba45b309d5234e95edc7ca5286fd", + "type": "Secret Keyword", + "filename": "tf_files/gcp/commons/root.tf", + "hashed_secret": "013b6be0bd7ef38a9ee3472cec65c208a19421e6", "is_verified": false, - "line_number": 11, - "type": "Secret Keyword" - }, + "line_number": 65 + } + ], + "tf_files/gcp/commons/sample.tfvars": [ { - "hashed_secret": "8db3b325254b6389ca194d829d2fc923dc0a945d", + "type": "Secret Keyword", + "filename": "tf_files/gcp/commons/sample.tfvars", + "hashed_secret": "6b44a330b450ee550c081410c6b705dfeaa105ce", "is_verified": false, - "line_number": 26, - "type": "Secret Keyword" + "line_number": 26 }, { - "hashed_secret": "253c7b5e7c83a86346fc4501495b130813f08105", - "is_verified": false, - "line_number": 37, - "type": "Secret Keyword" - } - ], - "tf_files/shared/modules/k8s_configs/creds.tpl": [ - { - "hashed_secret": "1f5e25be9b575e9f5d39c82dfd1d9f4d73f1975c", + "type": "Secret Keyword", + "filename": "tf_files/gcp/commons/sample.tfvars", + "hashed_secret": "791191ef9eafc75f5dd28e37df837b4991556876", "is_verified": false, - "line_number": 8, - "type": "Secret Keyword" + "line_number": 31 } ] }, - "version": "0.13.1", - "word_list": { - "file": null, - "hash": null - } + "generated_at": "2024-03-07T21:26:14Z" } diff --git a/files/scripts/ecr-access-job-requirements.txt b/files/scripts/ecr-access-job-requirements.txt new file mode 100644 index 0000000000..bb6d4b847d --- /dev/null +++ b/files/scripts/ecr-access-job-requirements.txt @@ -0,0 +1 @@ +boto3<2 diff --git a/files/scripts/ecr-access-job.md b/files/scripts/ecr-access-job.md new file mode 100644 index 0000000000..9659b186b9 --- /dev/null +++ b/files/scripts/ecr-access-job.md @@ -0,0 +1,85 @@ +# ecr-access-job + +### How to run + +Configure `global.ecr-access-job-role-arn` to the ARN of the `EcrRepoPolicyUpdateRole` role (described below) in the `manifest.json` file. + +Run `gen3 kube-setup-ecr-access-cronjob` to set up the ECR access cronjob. + +### What does it do? + +The job runs the `ecr-access-job.py` script. + +This script updates the configuration of ECR repositories so that users can access the repositories that were created for them. + +It queries a DynamoDB table which has the following (simplified) structure: +| user_id | workspace_type | account_id | +| ------------------ | -------------------- | ---------- | +| user1@username.com | Direct Pay | 123456 | +| user2@username.com | Direct Pay | 789012 | +| user1@username.com | Other workspace type | | + +and then allows each AWS account to acccess the appropriate ECR repositories. The users' ECR repositories are based on their username as stored in the table. For example, `user1@username.com`'s ECR repository is assumed to be `nextflow-approved/user1-40username-2ecom`. + +### Access needed + +- "EcrRepoPolicyUpdateRole" role in the account (Acct1) that contains the ECR repositories: + +**Note:** `kube-setup-ecr-access-cronjob.sh` assumes this role already exists. + +Permissions: +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UpdateEcrRepoPolicy", + "Effect": "Allow", + "Action": "ecr:SetRepositoryPolicy", + "Resource": "arn:aws:ecr:us-east-1::repository/nextflow-approved/*" + } + ] +} +``` + +Trust policy (allows Acct2): +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAssumingRole", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +- Policy in the account (Acct2) that contains the DynamoDB table (created automatically by `kube-setup-ecr-access-job.sh`): +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ReadDynamoDB", + "Effect": "Allow", + "Action": [ + "dynamodb:Scan" + ], + "Resource": "arn:aws:dynamodb:::table/" + }, + { + "Sid": "AssumeEcrRole", + "Effect": "Allow", + "Action": [ + "sts:AssumeRole" + ], + "Resource": "arn:aws:iam:::role/" + } + ] +} +``` diff --git a/files/scripts/ecr-access-job.py b/files/scripts/ecr-access-job.py new file mode 100644 index 0000000000..828d94c96b --- /dev/null +++ b/files/scripts/ecr-access-job.py @@ -0,0 +1,177 @@ +""" +See documentation at https://github.com/uc-cdis/cloud-automation/blob/master/files/scripts/ecr-access-job.md +""" + +from decimal import Decimal +import json +import os +from typing import List +import uuid + +import boto3 +from boto3.dynamodb.conditions import Attr + + +REGION = "us-east-1" + +# for local testing. in production, use a service account instead of a key. +MAIN_ACCOUNT_CREDS = {"key_id": os.environ.get("KEY_ID"), "key_secret": os.environ.get("KEY_SECRET")} + + +def escapism(string: str) -> str: + """ + This is a direct translation of Hatchery's `escapism` golang function to python. + We need to escape the username in the same way it's escaped by Hatchery's `escapism` function because + special chars cannot be used in an ECR repo name, and so that the ECR repo generated here matches the + name expected by Hatchery. + """ + safeBytes = "abcdefghijklmnopqrstuvwxyz0123456789" + escaped = "" + for v in string: + if v not in safeBytes: + hexCode = "{0:02x}".format(ord(v)) + escaped += "-" + hexCode + else: + escaped += v + return escaped + + +def get_configs() -> (str, str): + table_name = os.environ.get("PAY_MODELS_DYNAMODB_TABLE") + if not table_name: + raise Exception("Missing 'PAY_MODELS_DYNAMODB_TABLE' environment variable") + + ecr_role_arn = os.environ.get("ECR_ACCESS_JOB_ARN") + if not ecr_role_arn: + raise Exception("Missing 'ECR_ACCESS_JOB_ARN' environment variable") + + return table_name, ecr_role_arn + + +def query_usernames_and_account_ids(table_name: str) -> List[dict]: + """ + Returns: + List[dict]: [ { "user_id": "user1@username.com", "account_id": "123456" } ] + """ + if MAIN_ACCOUNT_CREDS["key_id"]: + session = boto3.Session( + aws_access_key_id=MAIN_ACCOUNT_CREDS["key_id"], + aws_secret_access_key=MAIN_ACCOUNT_CREDS["key_secret"], + ) + else: + session = boto3.Session() + dynamodb = session.resource("dynamodb", region_name=REGION) + table = dynamodb.Table(table_name) + + # get usernames and AWS account IDs from DynamoDB + queried_keys = ["user_id", "account_id"] + filter_expr = Attr("workspace_type").eq("Direct Pay") + proj = ", ".join("#" + key for key in queried_keys) + expr = {"#" + key: key for key in queried_keys} + response = table.scan( + FilterExpression=filter_expr, + ProjectionExpression=proj, + ExpressionAttributeNames=expr, + ) + assert response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200, response + items = response["Items"] + # if the response is paginated, get the rest of the items + while response["Count"] > 0: + if "LastEvaluatedKey" not in response: + break + response = table.scan( + FilterExpression=filter_expr, + ProjectionExpression=proj, + ExpressionAttributeNames=expr, + ExclusiveStartKey=response["LastEvaluatedKey"], + ) + assert ( + response.get("ResponseMetadata", {}).get("HTTPStatusCode") == 200 + ), response + items.extend(response["Items"]) + + return items + + +def update_access_in_ecr(repo_to_account_ids: List[dict], ecr_role_arn: str) -> None: + # get access to ECR in the account that contains the ECR repos + if MAIN_ACCOUNT_CREDS["key_id"]: + sts = boto3.client( + "sts", + aws_access_key_id=MAIN_ACCOUNT_CREDS["key_id"], + aws_secret_access_key=MAIN_ACCOUNT_CREDS["key_secret"], + ) + else: + sts = boto3.client("sts") + assumed_role = sts.assume_role( + RoleArn=ecr_role_arn, + DurationSeconds=900, # minimum time for aws assume role as per boto docs + RoleSessionName=f"ecr-access-assume-role-{str(uuid.uuid4())[:8]}", + ) + assert "Credentials" in assumed_role, "Unable to assume role" + ecr = boto3.client( + "ecr", + aws_access_key_id=assumed_role["Credentials"]["AccessKeyId"], + aws_secret_access_key=assumed_role["Credentials"]["SecretAccessKey"], + aws_session_token=assumed_role["Credentials"]["SessionToken"], + ) + + # for each ECR repo, whitelist the account IDs so users can access the repo + for repo, account_ids in repo_to_account_ids.items(): + print(f"Allowing AWS accounts {account_ids} to use ECR repository '{repo}'") + policy = { + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "AllowCrossAccountPull", + "Effect": "Allow", + "Principal": { + "AWS": [ + f"arn:aws:iam::{account_id}:root" + for account_id in account_ids + ] + }, + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetAuthorizationToken", + "ecr:GetDownloadUrlForLayer", + ], + } + ], + } + # Note that this is overwriting the repo policy, not appending to it. This means we can't have 2 dynamodb + # tables pointing at the same set of ECR repos: the repos would only allow the accounts in the table for + # which the script was run most recently. eg QA and Staging can't use the same ECR repos. + # Appending is not possible since this code will eventually rely on Arborist for authorization information + # and we'll need to overwrite in order to remove expired access. + try: + ecr.set_repository_policy( + repositoryName=repo, + policyText=json.dumps(policy), + ) + except Exception as e: + print(f" Unable to update '{repo}'; skipping it: {e}") + + +def main() -> None: + table_name, ecr_role_arn = get_configs() + items = query_usernames_and_account_ids(table_name) + + # construct mapping: { ECR repo url: [ AWS account IDs with access ] } + ecr_repo_prefix = "nextflow-approved" + repo_to_account_ids = { + f"{ecr_repo_prefix}/{escapism(e['user_id'])}": [e["account_id"]] + for e in items + if "account_id" in e + } + print( + "Mapping of ECR repository to allowed AWS accounts:\n", + json.dumps(repo_to_account_ids, indent=2), + ) + + update_access_in_ecr(repo_to_account_ids, ecr_role_arn) + + +if __name__ == "__main__": + main() diff --git a/gen3/bin/kube-setup-ecr-access-cronjob.sh b/gen3/bin/kube-setup-ecr-access-cronjob.sh new file mode 100644 index 0000000000..d23afc862c --- /dev/null +++ b/gen3/bin/kube-setup-ecr-access-cronjob.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +source "${GEN3_HOME}/gen3/lib/utils.sh" +gen3_load "gen3/lib/kube-setup-init" + +setup_ecr_access_job() { + if g3kubectl get configmap manifest-global > /dev/null; then + ecrRoleArn=$(g3kubectl get configmap manifest-global -o jsonpath={.data.ecr-access-job-role-arn}) + fi + if [ -z "$ecrRoleArn" ]; then + gen3_log_err "Missing 'global.ecr-access-job-role-arn' configuration in manifest.json" + return 1 + fi + + local saName="ecr-access-job-sa" + if ! g3kubectl get sa "$saName" > /dev/null 2>&1; then + tempFile="ecr-access-job-policy.json" + cat - > $tempFile < Date: Fri, 8 Mar 2024 16:28:46 -0600 Subject: [PATCH 078/114] Add new middleware url (#2497) * add new middleware url * feat(argo-wrapper): newline --------- Co-authored-by: Andrew Prokhorenkov --- kube/services/argo-wrapper/config.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/kube/services/argo-wrapper/config.ini b/kube/services/argo-wrapper/config.ini index 40ac392fd5..0693ee2e29 100644 --- a/kube/services/argo-wrapper/config.ini +++ b/kube/services/argo-wrapper/config.ini @@ -3,3 +3,4 @@ ARGO_ACCESS_METHOD = access ARGO_HOST = $ARGO_HOST ARGO_NAMESPACE = $ARGO_NAMESPACE COHORT_DEFINITION_BY_SOURCE_AND_TEAM_PROJECT_URL = http://cohort-middleware-service/cohortdefinition-stats/by-source-id/{}/by-team-project?team-project={} +COHORT_MIDDLEWARE_URL = http://cohort-middleware-service From f84581c1fd8899d7cbb09d48464bb6a9a6ceaa47 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 11 Mar 2024 09:35:52 -0600 Subject: [PATCH 079/114] Update gen3-helpers to fix failing tests (#2443) * updating the awsrole script to better handle variables and flag * adding "migrate to vpc cni" script to cloud-auto * removing script- wrong branch * updating kube-setup-argo script for IRSA * changing the script to get the 3 private ip to account for fargate instances and fix the "ec2 test filter" test * changing -S to -e to verify if docker is the container runtime due to the way that Jenkins pods are setup * testing mount change * testing another mount change * reverting change * mounting containerd instead * modifying jenkins shell commands * reverting ecr change * reverting jenkisfile change and then changing permissions on containerd.sock instead of docker.sock * adding more jobs to the "excludeJob" list * commenting out jupyter metric test as we currently don't use prometheus in most environments * commented out the wrong line in the wrong script * removing terraform test as we are now using Atlantis and Terragrunt to manage our infastructure and will no longer use the "workon" command * fixing snapshot script to grab init containers and all the main containers seperately * updating to use "pip3" instead of /usr/bin/pip3" * changing the evicted pod cleanup to produce no output so the healthcheck will pass in Jenkins. also, adding in the proper path for pip in the pytest stage of the pipeline * trying another method to get rid of the json error when running the gen3 healthcheck test * Update healthcheck.sh moving clear_evicted_pods function call to run inside the healthcheck function --- Jenkinsfile | 12 +- gen3/bin/awsrole.sh | 48 ++-- gen3/bin/healthcheck.sh | 6 +- gen3/bin/kube-setup-argo.sh | 2 +- gen3/lib/logs/snapshot.sh | 2 +- gen3/test/ec2Test.sh | 2 +- gen3/test/ecrTest.sh | 4 +- gen3/test/jobTest.sh | 2 +- gen3/test/jupyterTest.sh | 2 +- gen3/test/terraformTest.sh | 461 ------------------------------------ 10 files changed, 39 insertions(+), 502 deletions(-) delete mode 100644 gen3/test/terraformTest.sh diff --git a/Jenkinsfile b/Jenkinsfile index 9c70a2e378..908c2d01a5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -134,8 +134,8 @@ spec: readOnly: true mountPath: "/usr/local/share/ca-certificates/cdis/cdis-ca.crt" subPath: "ca.pem" - - name: dockersock - mountPath: "/var/run/docker.sock" + - name: containerdsock + mountPath: "/var/run/containerd/containerd.sock" serviceAccount: jenkins-service serviceAccountName: jenkins-service volumes: @@ -145,9 +145,9 @@ spec: - name: ca-volume secret: secretName: "service-ca" - - name: dockersock + - name: containerdsock hostPath: - path: /var/run/docker.sock + path: /var/run/containerd/containerd.sock ''' defaultContainer 'shell' } @@ -293,8 +293,8 @@ spec: script { try { if(!skipUnitTests) { - sh '/usr/bin/pip3 install boto3 --upgrade --user' - sh '/usr/bin/pip3 install kubernetes --upgrade --user' + sh '/usr/local/bin/pip3 install boto3 --upgrade --user' + sh '/usr/local/bin/pip3 install kubernetes --upgrade --user' sh 'python3 -m pytest cloud-automation/apis_configs/' sh 'python3 -m pytest cloud-automation/gen3/lib/dcf/' sh 'cd cloud-automation/tf_files/aws/modules/common-logging && python3 -m pytest testLambda.py' diff --git a/gen3/bin/awsrole.sh b/gen3/bin/awsrole.sh index 144b7a4fea..dd19ea7a48 100644 --- a/gen3/bin/awsrole.sh +++ b/gen3/bin/awsrole.sh @@ -25,16 +25,16 @@ gen3_awsrole_help() { function gen3_awsrole_ar_policy() { local serviceAccount="$1" shift || return 1 - if [[ ! -z $1 ]]; then - local namespace=$1 + if [[ -z $1 ]] || [[ $1 == -* ]]; then + namespace=$(gen3 db namespace) else - local namespace=$(gen3 db namespace) + namespace=$1 + shift fi local issuer_url local account_id local vpc_name - shift || return 1 - local flag=$1 + local flag=$flag vpc_name="$(gen3 api environment)" || return 1 issuer_url="$(aws eks describe-cluster \ @@ -46,7 +46,7 @@ function gen3_awsrole_ar_policy() { local provider_arn="arn:aws:iam::${account_id}:oidc-provider/${issuer_url}" - if [[ "$flag" == "all_namespaces" ]]; then + if [[ "$flag" == "-all_namespaces" ]]; then # Use a trust policy that allows role to be used by multiple namespaces. cat - < config.tfvars @@ -230,10 +226,14 @@ gen3_awsrole_create() { gen3_log_err "use: gen3 awsrole create roleName saName" return 1 fi - if [[ ! -z $1 ]]; then - local namespace=$1 + if [[ -z $1 ]] || [[ $1 == -* ]]; then + namespace=$(gen3 db namespace) else - local namespace=$(gen3 db namespace) + namespace=$1 + shift + fi + if [[ ! -z $1 ]]; then + flag=$1 fi # do simple validation of name local regexp="^[a-z][a-z0-9\-]*$" @@ -247,13 +247,7 @@ EOF gen3_log_err $errMsg return 1 fi - shift || return 1 - local flag="" - # Check if the "all_namespaces" flag is provided - if [[ "$1" == "-f" || "$1" == "--flag" ]]; then - flag="$2" - shift 2 - fi + # check if the name is already used by another entity local entity_type @@ -271,9 +265,11 @@ EOF fi TF_IN_AUTOMATION="true" - if ! _tfplan_role $rolename $saName $namespace -f $flag; then + + if ! _tfplan_role $rolename $saName $namespace $flag; then return 1 fi + if ! _tfapply_role $rolename; then return 1 fi @@ -422,4 +418,4 @@ gen3_awsrole() { # Let testsuite source file if [[ -z "$GEN3_SOURCE_ONLY" ]]; then gen3_awsrole "$@" -fi +fi \ No newline at end of file diff --git a/gen3/bin/healthcheck.sh b/gen3/bin/healthcheck.sh index 149cb1aaa9..b658ff033b 100644 --- a/gen3/bin/healthcheck.sh +++ b/gen3/bin/healthcheck.sh @@ -137,6 +137,10 @@ gen3_healthcheck() { internetAccessExplicitProxy=false fi + gen3_log_info "Clearing Evicted pods" + sleep 5 + clear_evicted_pods + local healthJson=$(cat - < ". g3kubectl get pods -o json | \ - jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | map( {pod: .metadata.name, containers: [(.spec.containers | select(.!=null) | map(.name)), (.spec.initContainers | select(.!=null) | map(.name)) | add ] } ) | map( .pod as $pod | .containers | map( { pod: $pod, cont: .})[]) | map(select(.cont != "pause" and .cont != "jupyterhub"))[] | .pod + " " + .cont' | \ + jq -r '.items | map(select(.status.phase != "Pending" and .status.phase != "Unknown")) | .[] | .metadata.name as $pod | (.spec.containers + .spec.initContainers) | map(select(.name != "pause" and .name != "jupyterhub")) | .[] | {pod: $pod, cont: .name} | "\(.pod) \(.cont)"' | \ while read -r line; do gen3_logs_snapshot_container $line done diff --git a/gen3/test/ec2Test.sh b/gen3/test/ec2Test.sh index 21310a24ca..4981c925cf 100644 --- a/gen3/test/ec2Test.sh +++ b/gen3/test/ec2Test.sh @@ -1,6 +1,6 @@ -if ! EC2_TEST_IP="$(g3kubectl get nodes -o json | jq -r -e '.items[0].status.addresses[] | select(.type == "InternalIP") | .address')" || [[ -z "$EC2_TEST_IP" ]]; then +if ! EC2_TEST_IP="$(g3kubectl get nodes -o json | jq -r -e '.items[3].status.addresses[] | select(.type == "InternalIP") | .address')" || [[ -z "$EC2_TEST_IP" ]]; then gen3_log_err "ec2Test failed to acquire IP address of a k8s node to test against" fi diff --git a/gen3/test/ecrTest.sh b/gen3/test/ecrTest.sh index 91edf798ba..57847abe5e 100644 --- a/gen3/test/ecrTest.sh +++ b/gen3/test/ecrTest.sh @@ -10,8 +10,8 @@ test_ecr_login() { test_ecr_setup() { if [[ -n "$JENKINS_HOME" ]]; then - # give ourselves read/write permissions on /var/run/docker.sock - sudo chmod a+rw /var/run/docker.sock; because $? "ecr_setup modified docker.sock" + # give ourselves permissions on /run/containerd/containerd.sock + sudo chown root:sudo /run/containerd/containerd.sock; because $? "ecr_setup modified containerd.sock" fi } diff --git a/gen3/test/jobTest.sh b/gen3/test/jobTest.sh index 84a4d046b6..bb37b4f723 100644 --- a/gen3/test/jobTest.sh +++ b/gen3/test/jobTest.sh @@ -6,7 +6,7 @@ excludeJob() { local jobKey="$1" local excludeList=( - /aws-bucket- /bucket- /covid19- /data-ingestion- /google- /nb-etl- /remove-objects-from- /replicate- /s3sync- /fence-cleanup + /aws-bucket- /bucket- /covid19- /data-ingestion- /google- /nb-etl- /remove-objects-from- /replicate- /s3sync- /fence-cleanup /etl- /indexd- /metadata- ) for exclude in "${excludeList[@]}"; do if [[ "$it" =~ $exclude ]]; then return 0; fi diff --git a/gen3/test/jupyterTest.sh b/gen3/test/jupyterTest.sh index f0e327d717..db6a626188 100644 --- a/gen3/test/jupyterTest.sh +++ b/gen3/test/jupyterTest.sh @@ -30,7 +30,7 @@ test_jupyter_metrics() { } shunit_runtest "test_jupyter_idle" "jupyter" -shunit_runtest "test_jupyter_metrics" "jupyter" +# shunit_runtest "test_jupyter_metrics" "jupyter" shunit_runtest "test_jupyter_prepuller" "local,jupyter" shunit_runtest "test_jupyter_namespace" "local,jupyter" shunit_runtest "test_jupyter_setup" "jupyter" diff --git a/gen3/test/terraformTest.sh b/gen3/test/terraformTest.sh deleted file mode 100644 index 17bcc03c2b..0000000000 --- a/gen3/test/terraformTest.sh +++ /dev/null @@ -1,461 +0,0 @@ -GEN3_TEST_PROFILE="${GEN3_TEST_PROFILE:-cdistest}" -GEN3_TEST_WORKSPACE="gen3test" -GEN3_TEST_ACCOUNT=707767160287 - -# -# TODO - generalize these tests to setup their own test VPC, -# rather than relying on qaplanetv1 or devplanetv1 being there -# - -# -# Little macos/linux stat wrapper -# -file_mode() { - if [[ $(uname -s) == 'Linux' ]]; then - stat -c %a "$1" - else - stat -f %p "$1" - fi -} - -test_workspace() { - gen3 workon $GEN3_TEST_PROFILE $GEN3_TEST_WORKSPACE; because $? "Calling gen3 workon multiple times should be harmless" - [[ $GEN3_PROFILE = $GEN3_TEST_PROFILE ]]; because $? "gen3 workon sets the GEN3_PROFILE env variable: $GEN3_PROFILE" - [[ $GEN3_WORKSPACE = $GEN3_TEST_WORKSPACE ]]; because $? "gen3 workon sets the GEN3_WORKSPACE env variable: $GEN3_WORKSPACE" - [[ $GEN3_FLAVOR = "AWS" || \ - ($GEN3_FLAVOR == "GCP" && $GEN3_PROFILE =~ ^gcp-) || \ - ($GEN3_FLAVOR == "ONPREM" && $GEN3_PROFILE =~ ^onprem-) ]]; because $? "GEN3_FLAVOR is gcp for gcp-* profiles, else AWS" - [[ $GEN3_FLAVOR != "AWS" || $GEN3_S3_BUCKET = "cdis-state-ac${GEN3_TEST_ACCOUNT}-gen3" || $GEN3_S3_BUCKET = "cdis-terraform-state.account-${GEN3_TEST_ACCOUNT}.gen3" ]]; because $? "gen3 workon sets the GEN3_S3_BUCKET env variable: $GEN3_S3_BUCKET" - [[ (! -z $GEN3_WORKDIR) && -d $GEN3_WORKDIR ]]; because $? "gen3 workon sets the GEN3_WORKDIR env variable, and initializes the folder: $GEN3_WORKDIR" - [[ $(file_mode $GEN3_WORKDIR) =~ 700$ ]]; because $? "gen3 workon sets the GEN3_WORKDIR to mode 0700, because secrets are in there" - gen3 cd && [[ $(pwd) = "$GEN3_WORKDIR" ]]; because $? "gen3 cd should take us to the workspace by default: $(pwd) =? $GEN3_WORKDIR" - for fileName in README.md config.tfvars backend.tfvars; do - [[ -f $fileName ]]; because $? "gen3 workon ensures we have a $fileName - local copy || s3 copy || generated from template" - done - [[ ! -z "$MD5" ]]; because $? "commons.sh sets MD5 to $MD5" - - if [[ $GEN3_TEST_WORKSPACE =~ __custom$ ]]; then - [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_WORKDIR" ]]; because $? "a __custom workspace loads from the workspace folder" - elif [[ "$GEN3_TEST_PROFILE" =~ ^gcp- ]]; then - [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/gcp/commons" ]]; because $? "a gcp- profile currently only support a commons workspace" - elif [[ "$GEN3_TEST_PROFILE" =~ ^onprem- ]]; then - for fileName in README.md creds.json 00configmap.yaml kube-setup.sh; do - filePath="onprem_scripts/$fileName" - [[ -f $filePath ]]; because $? "gen3 workon ensures we have a $filePath generated from template" - done - else # aws profile - [[ "$GEN3_TFSCRIPT_FOLDER" =~ ^"$GEN3_HOME/tf_files/aws/" ]]; because $? "an aws workspace references the aws/ folder: $GEN3_TFSCRIPT_FOLDER" - fi -} - -workspace_cleanup() { - # try to avoid accidentally erasing the user's data ... - cd /tmp && [[ -n "$GEN3_WORKDIR" && "$GEN3_WORKDIR" =~ /gen3/ && -f "$GEN3_WORKDIR/config.tfvars" ]] && /bin/rm -rf "$GEN3_WORKDIR"; - because $? "was able to cleanup $GEN3_WORKDIR" -} - -test_uservpc_workspace() { - GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_user" - test_workspace - [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/user_vpc" ]]; because $? "a _user workspace should use the ./aws/user_vpc resources: $GEN3_TFSCRIPT_FOLDER" - workspace_cleanup -} - -test_usergeneric_workspace() { - GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_usergeneric" - test_workspace - [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/user_generic" ]]; because $? "a _usergeneric workspace should use the ./aws/user_generic resources: $GEN3_TFSCRIPT_FOLDER" - cat << EOF > config.tfvars -username="frickjack" -EOF - gen3 tfplan; because $? "_usergeneric tfplan should work"; - workspace_cleanup -} - -test_snapshot_workspace() { - GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_snapshot" - test_workspace - [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/rds_snapshot" ]]; because $? "a _snapshot workspace should use the ./aws/rds_snapshot resources: $GEN3_TFSCRIPT_FOLDER" - workspace_cleanup -} - -test_databucket_workspace() { - GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}_databucket" - test_workspace - [[ "$GEN3_TFSCRIPT_FOLDER" == "$GEN3_HOME/tf_files/aws/data_bucket" ]]; because $? "a _databucket workspace should use the ./aws/data_bucket resources: $GEN3_TFSCRIPT_FOLDER" - cat - > config.tfvars < config.tfvars < config.tfvars < @ in password -db_password_fence="whatever" - -db_password_gdcapi="whatever" -db_password_sheepdog="whatever" -db_password_peregrine="whatever" - -db_password_indexd="g6pmYkcoR7qECjGoErzVb5gkX3kum0yo" - -# password for write access to indexd -gdcapi_indexd_password="oYva39mIPV5uXskv7jWnKuVZBUFBQcxd" - -fence_snapshot="" -gdcapi_snapshot="" -indexd_snapshot="" -# mailgun for sending alert e-mails -mailgun_api_key="" -mailgun_api_url="" -mailgun_smtp_host="" - -kube_ssh_key="" -EOM - [[ "$(pwd)" =~ "/$GEN3_WORKSPACE"$ ]]; because $? "commons workspace should have base $GEN3_WORKSPACE - $(pwd)" - gen3 tfplan; because $? "tfplan should run even with some invalid config variables" - [[ -f "$GEN3_WORKDIR/plan.terraform" ]]; because $? "'gen3 tfplan' generates a plan.terraform file used by 'gen3 tfapply'" - workspace_cleanup -} - -test_custom_workspace() { - GEN3_TEST_WORKSPACE="${GEN3_TEST_WORKSPACE}__custom" - test_workspace - - local sourceFolder="../../../../../cloud-automation/tf_files/aws/modules/s3-bucket" - if [[ ! -d "$sourceFolder" ]]; then - # Jenkins has a different relative path setup - sourceFolder="../../../../cloud-automation/tf_files/aws/modules/s3-bucket" - fi - cat - > bucket.tf < config.tfvars < config.tfvars < config.tfvars < config.tfvars < config.tfvars < config.tfvars < config.tfvars < config.tfvars < Date: Tue, 12 Mar 2024 10:05:49 -0500 Subject: [PATCH 080/114] Split build-push workflows (#2499) --- .github/workflows/build_awshelper.yaml | 21 +++++++++++++ .github/workflows/build_python3.10.yaml | 23 ++++++++++++++ .github/workflows/build_python3.9.yaml | 23 ++++++++++++++ .github/workflows/image_build_push.yaml | 42 ------------------------- 4 files changed, 67 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/build_awshelper.yaml create mode 100644 .github/workflows/build_python3.10.yaml create mode 100644 .github/workflows/build_python3.9.yaml delete mode 100644 .github/workflows/image_build_push.yaml diff --git a/.github/workflows/build_awshelper.yaml b/.github/workflows/build_awshelper.yaml new file mode 100644 index 0000000000..f42a136109 --- /dev/null +++ b/.github/workflows/build_awshelper.yaml @@ -0,0 +1,21 @@ +name: Build awshelper image + +on: + push: + paths: + - .github/workflows/build_awshelper.yaml + - Docker/awshelper/** + +jobs: + awshelper: + name: Build and push + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile" + OVERRIDE_REPO_NAME: "awshelper" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + diff --git a/.github/workflows/build_python3.10.yaml b/.github/workflows/build_python3.10.yaml new file mode 100644 index 0000000000..993da14680 --- /dev/null +++ b/.github/workflows/build_python3.10.yaml @@ -0,0 +1,23 @@ +name: Build Python 3.10 image + +on: + push: + paths: + - .github/workflows/build_python3.10.yaml + - Docker/python-nginx/python3.10-buster/** + +jobs: + python_3-10: + name: Build and push + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.10-buster" + OVERRIDE_REPO_NAME: "python" + OVERRIDE_TAG_NAME: "python3.10-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + diff --git a/.github/workflows/build_python3.9.yaml b/.github/workflows/build_python3.9.yaml new file mode 100644 index 0000000000..5bc8bc4629 --- /dev/null +++ b/.github/workflows/build_python3.9.yaml @@ -0,0 +1,23 @@ +name: Build Python 3.9 image + +on: + push: + paths: + - .github/workflows/build_python3.9.yaml + - Docker/python-nginx/python3.9-buster/** + +jobs: + python_3-9: + name: Build and push + uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master + with: + DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile" + DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.9-buster" + OVERRIDE_REPO_NAME: "python" + OVERRIDE_TAG_NAME: "python3.9-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)" + secrets: + ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} + ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} + diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml deleted file mode 100644 index d5bfea351d..0000000000 --- a/.github/workflows/image_build_push.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: Build Python Base Images - -on: push - -jobs: - python_3-9: - name: Python 3.9 - uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master - with: - DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile" - DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.9-buster" - OVERRIDE_REPO_NAME: "python" - OVERRIDE_TAG_NAME: "python3.9-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)" - secrets: - ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} - ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} - python_3-10: - name: Python 3.10 - uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master - with: - DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile" - DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.10-buster" - OVERRIDE_REPO_NAME: "python" - OVERRIDE_TAG_NAME: "python3.10-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)" - secrets: - ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} - ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} - awshelper: - name: AwsHelper - uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master - with: - DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile" - OVERRIDE_REPO_NAME: "awshelper" - secrets: - ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }} - ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }} - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }} From b2195b1ae6f9d802d346629d126e9d3ba408eccd Mon Sep 17 00:00:00 2001 From: Pauline <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 12 Mar 2024 10:07:34 -0500 Subject: [PATCH 081/114] fix wf naming --- .github/workflows/build_awshelper.yaml | 2 +- .github/workflows/build_python3.10.yaml | 2 +- .github/workflows/build_python3.9.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_awshelper.yaml b/.github/workflows/build_awshelper.yaml index f42a136109..3d2da5393e 100644 --- a/.github/workflows/build_awshelper.yaml +++ b/.github/workflows/build_awshelper.yaml @@ -8,7 +8,7 @@ on: jobs: awshelper: - name: Build and push + name: awshelper uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/awshelper/Dockerfile" diff --git a/.github/workflows/build_python3.10.yaml b/.github/workflows/build_python3.10.yaml index 993da14680..80d2d76232 100644 --- a/.github/workflows/build_python3.10.yaml +++ b/.github/workflows/build_python3.10.yaml @@ -8,7 +8,7 @@ on: jobs: python_3-10: - name: Build and push + name: Python 3.10 uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile" diff --git a/.github/workflows/build_python3.9.yaml b/.github/workflows/build_python3.9.yaml index 5bc8bc4629..540e0d4eca 100644 --- a/.github/workflows/build_python3.9.yaml +++ b/.github/workflows/build_python3.9.yaml @@ -8,7 +8,7 @@ on: jobs: python_3-9: - name: Build and push + name: Python 3.9 uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master with: DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile" From b6031e029db84ab0190d2a263c16b418b113482d Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 12 Mar 2024 12:03:41 -0500 Subject: [PATCH 082/114] Build awshelper workflow: always build (#2501) --- .github/workflows/build_awshelper.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_awshelper.yaml b/.github/workflows/build_awshelper.yaml index 3d2da5393e..36b5745dbd 100644 --- a/.github/workflows/build_awshelper.yaml +++ b/.github/workflows/build_awshelper.yaml @@ -1,10 +1,8 @@ name: Build awshelper image -on: - push: - paths: - - .github/workflows/build_awshelper.yaml - - Docker/awshelper/** +# Always build this image because it contains all the cloud-automation files. +# Some jobs depend on arbitrary files and we need to test them with updated awshelper images. +on: push jobs: awshelper: From 24492c2d6868ce49a474617544b575a38697d0af Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Tue, 12 Mar 2024 16:23:59 -0400 Subject: [PATCH 083/114] Adding a gen3 db namespace to the temp files so they don't overlap (#2502) --- gen3/bin/kube-setup-revproxy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gen3/bin/kube-setup-revproxy.sh b/gen3/bin/kube-setup-revproxy.sh index 5db9850a18..fd30b478b3 100644 --- a/gen3/bin/kube-setup-revproxy.sh +++ b/gen3/bin/kube-setup-revproxy.sh @@ -114,8 +114,8 @@ done if g3k_manifest_lookup .argo.argo_server_service_url 2> /dev/null; then argo_server_service_url=$(g3k_manifest_lookup .argo.argo_server_service_url) - g3k_kv_filter "${scriptDir}/gen3.nginx.conf/argo-server.conf" SERVICE_URL "${argo_server_service_url}" > /tmp/argo-server-with-url.conf - filePath="/tmp/argo-server-with-url.conf" + g3k_kv_filter "${scriptDir}/gen3.nginx.conf/argo-server.conf" SERVICE_URL "${argo_server_service_url}" > /tmp/argo-server-with-url$(gen3 db namespace).conf + filePath="/tmp/argo-server-with-url$(gen3 db namespace).conf" if [[ -f "$filePath" ]]; then confFileList+=("--from-file" "$filePath") fi From e979669cd92cf1ecad69f2bc2837a8fed35e2926 Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Wed, 13 Mar 2024 12:08:30 -0500 Subject: [PATCH 084/114] HP-1310 feat: updated related studies logic (#2498) * feat: updated related studies logic * update --- .../healdata/heal-cedar-data-ingest.py | 174 +++++++++++------- 1 file changed, 107 insertions(+), 67 deletions(-) diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index c54f9d5aa0..e0c4b3c463 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -13,7 +13,7 @@ "study_metadata.study_type.study_subject_type": "Subject Type", "study_metadata.human_subject_applicability.gender_applicability": "Gender", "study_metadata.human_subject_applicability.age_applicability": "Age", - "research_program": "Research Program" + "research_program": "Research Program", } # Defines how to handle special cases for values in filters @@ -33,7 +33,7 @@ "Gender Queer": "Genderqueer/gender nonconforming/neither exclusively male nor female", "Intersex": "Genderqueer/gender nonconforming/neither exclusively male nor female", "Intersexed": "Genderqueer/gender nonconforming/neither exclusively male nor female", - "Buisness Development": "Business Development" + "Buisness Development": "Business Development", } # Defines field that we don't want to include in the filters @@ -54,24 +54,25 @@ def is_valid_uuid(uuid_to_test, version=4): """ Check if uuid_to_test is a valid UUID. - + Parameters ---------- uuid_to_test : str version : {1, 2, 3, 4} - + Returns ------- `True` if uuid_to_test is a valid UUID, otherwise `False`. - + """ - + try: uuid_obj = UUID(uuid_to_test, version=version) except ValueError: return False return str(uuid_obj) == uuid_to_test + def update_filter_metadata(metadata_to_update): filter_metadata = [] for metadata_field_key, filter_field_key in FILTER_FIELD_MAPPINGS.items(): @@ -83,20 +84,21 @@ def update_filter_metadata(metadata_to_update): print(filter_field_values) raise TypeError("Neither a string nor a list") for filter_field_value in filter_field_values: - if (metadata_field_key, filter_field_value) in OMITTED_VALUES_MAPPING.items(): + if ( + metadata_field_key, + filter_field_value, + ) in OMITTED_VALUES_MAPPING.items(): continue if filter_field_value in SPECIAL_VALUE_MAPPINGS: filter_field_value = SPECIAL_VALUE_MAPPINGS[filter_field_value] - filter_metadata.append({"key": filter_field_key, "value": filter_field_value}) + filter_metadata.append( + {"key": filter_field_key, "value": filter_field_value} + ) filter_metadata = pydash.uniq(filter_metadata) metadata_to_update["advSearchFilters"] = filter_metadata # Retain these from existing tags save_tags = ["Data Repository"] - tags = [ - tag - for tag in metadata_to_update["tags"] - if tag["category"] in save_tags - ] + tags = [tag for tag in metadata_to_update["tags"] if tag["category"] in save_tags] # Add any new tags from advSearchFilters for f in metadata_to_update["advSearchFilters"]: if f["key"] == "Gender": @@ -111,25 +113,30 @@ def update_filter_metadata(metadata_to_update): def get_client_token(client_id: str, client_secret: str): try: token_url = f"http://revproxy-service/user/oauth2/token" - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - params = {'grant_type': 'client_credentials'} - data = 'scope=openid user data' + headers = {"Content-Type": "application/x-www-form-urlencoded"} + params = {"grant_type": "client_credentials"} + data = "scope=openid user data" token_result = requests.post( - token_url, params=params, headers=headers, data=data, + token_url, + params=params, + headers=headers, + data=data, auth=(client_id, client_secret), ) - token = token_result.json()["access_token"] + token = token_result.json()["access_token"] except: raise Exception("Could not get token") return token -def get_related_studies(serial_num, hostname): +def get_related_studies(serial_num, guid, hostname): related_study_result = [] if serial_num: - mds = requests.get(f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000") + mds = requests.get( + f"http://revproxy-service/mds/metadata?nih_reporter.project_num_split.serial_num={serial_num}&data=true&limit=2000" + ) if mds.status_code == 200: related_study_metadata = mds.json() @@ -137,15 +144,22 @@ def get_related_studies(serial_num, hostname): related_study_metadata_key, related_study_metadata_value, ) in related_study_metadata.items(): + if related_study_metadata_key == guid or ( + related_study_metadata_value["_guid_type"] != "discovery_metadata" + and related_study_metadata_value["_guid_type"] + != "unregistered_discovery_metadata" + ): + # do nothing for self, or for archived studies + continue title = ( - related_study_metadata_value.get( - "gen3_discovery", {} - ) + related_study_metadata_value.get("gen3_discovery", {}) .get("study_metadata", {}) .get("minimal_info", {}) .get("study_name", "") ) - link = f"https://{hostname}/portal/discovery/{related_study_metadata_key}/" + link = ( + f"https://{hostname}/portal/discovery/{related_study_metadata_key}/" + ) related_study_result.append({"title": title, "link": link}) return related_study_result @@ -180,7 +194,7 @@ def get_related_studies(serial_num, hostname): print("Getting CEDAR client access token") access_token = get_client_token(client_id, client_secret) -token_header = {"Authorization": 'bearer ' + access_token} +token_header = {"Authorization": "bearer " + access_token} limit = 10 offset = 0 @@ -192,16 +206,21 @@ def get_related_studies(serial_num, hostname): print("Directory ID is not in UUID format!") sys.exit(1) -while((limit + offset <= total)): +while limit + offset <= total: # Get the metadata from cedar to register print("Querying CEDAR...") - cedar = requests.get(f"http://revproxy-service/cedar/get-instance-by-directory/{dir_id}?limit={limit}&offset={offset}", headers=token_header) + cedar = requests.get( + f"http://revproxy-service/cedar/get-instance-by-directory/{dir_id}?limit={limit}&offset={offset}", + headers=token_header, + ) # If we get metadata back now register with MDS if cedar.status_code == 200: metadata_return = cedar.json() if "metadata" not in metadata_return: - print("Got 200 from CEDAR wrapper but no metadata in body, something is not right!") + print( + "Got 200 from CEDAR wrapper but no metadata in body, something is not right!" + ) sys.exit(1) total = metadata_return["metadata"]["totalCount"] @@ -209,13 +228,17 @@ def get_related_studies(serial_num, hostname): print(f"Successfully got {returned_records} record(s) from CEDAR directory") for cedar_record in metadata_return["metadata"]["records"]: # get the appl id from cedar for querying in our MDS - cedar_appl_id = pydash.get(cedar_record, "metadata_location.nih_application_id") + cedar_appl_id = pydash.get( + cedar_record, "metadata_location.nih_application_id" + ) if cedar_appl_id is None: print("This record doesn't have appl_id, skipping...") continue # Get the metadata record for the nih_application_id - mds = requests.get(f"http://revproxy-service/mds/metadata?gen3_discovery.study_metadata.metadata_location.nih_application_id={cedar_appl_id}&data=true") + mds = requests.get( + f"http://revproxy-service/mds/metadata?gen3_discovery.study_metadata.metadata_location.nih_application_id={cedar_appl_id}&data=true" + ) if mds.status_code == 200: mds_res = mds.json() @@ -234,9 +257,13 @@ def get_related_studies(serial_num, hostname): if mds_res["_guid_type"] == "discovery_metadata": print("Metadata is already registered. Updating MDS record") elif mds_res["_guid_type"] == "unregistered_discovery_metadata": - print("Metadata has not been registered. Registering it in MDS record") + print( + "Metadata has not been registered. Registering it in MDS record" + ) else: - print(f"This metadata data record has a special GUID type \"{mds_res['_guid_type']}\" and will be skipped") + print( + f"This metadata data record has a special GUID type \"{mds_res['_guid_type']}\" and will be skipped" + ) continue if "clinicaltrials_gov" in cedar_record: @@ -244,21 +271,27 @@ def get_related_studies(serial_num, hostname): del cedar_record["clinicaltrials_gov"] # some special handing for this field, because its parent will be deleted before we merging the CEDAR and MDS SLMD to avoid duplicated values - cedar_record_other_study_websites = cedar_record.get("metadata_location", {}).get("other_study_websites", []) + cedar_record_other_study_websites = cedar_record.get( + "metadata_location", {} + ).get("other_study_websites", []) del cedar_record["metadata_location"] mds_res["gen3_discovery"]["study_metadata"].update(cedar_record) - mds_res["gen3_discovery"]["study_metadata"]["metadata_location"]["other_study_websites"] = cedar_record_other_study_websites + mds_res["gen3_discovery"]["study_metadata"]["metadata_location"][ + "other_study_websites" + ] = cedar_record_other_study_websites # setup citations - doi_citation = mds_res["gen3_discovery"]["study_metadata"].get("doi_citation", "") - mds_res["gen3_discovery"]["study_metadata"]["citation"]["heal_platform_citation"] = doi_citation - + doi_citation = mds_res["gen3_discovery"]["study_metadata"].get( + "doi_citation", "" + ) + mds_res["gen3_discovery"]["study_metadata"]["citation"][ + "heal_platform_citation" + ] = doi_citation # setup repository_study_link data_repositories = ( - mds_res - .get("gen3_discovery", {}) + mds_res.get("gen3_discovery", {}) .get("study_metadata", {}) .get("metadata_location", {}) .get("data_repositories", []) @@ -275,8 +308,13 @@ def get_related_studies(serial_num, hostname): repository_study_link = REPOSITORY_STUDY_ID_LINK_TEMPLATE[ repository["repository_name"] ].replace("", repository["repository_study_ID"]) - repository.update({"repository_study_link": repository_study_link}) - if repository_citation_additional_text not in repository_citation: + repository.update( + {"repository_study_link": repository_study_link} + ) + if ( + repository_citation_additional_text + not in repository_citation + ): repository_citation += repository_citation_additional_text if len(data_repositories): data_repositories[0] = { @@ -284,36 +322,28 @@ def get_related_studies(serial_num, hostname): "repository_citation": repository_citation, } - mds_res["gen3_discovery"]["study_metadata"][ - "metadata_location" - ]["data_repositories"] = copy.deepcopy(data_repositories) - - + mds_res["gen3_discovery"]["study_metadata"]["metadata_location"][ + "data_repositories" + ] = copy.deepcopy(data_repositories) # set up related studies serial_num = None try: serial_num = ( - mds_res - .get("nih_reporter", {}) + mds_res.get("nih_reporter", {}) .get("project_num_split", {}) .get("serial_num", None) ) except Exception: - print(f"Unable to get serial number for study") - - if serial_num == None: - print(f"Unable to get serial number for study") + print("Unable to get serial number for study") - related_study_result = get_related_studies(serial_num, hostname) - existing_related_study_result = mds_res.get("related_studies", []) - for related_study in related_study_result: - if related_study not in existing_related_study_result: - existing_related_study_result.append(copy.deepcopy(related_study)) - mds_res["gen3_discovery"][ - "related_studies" - ] = copy.deepcopy(existing_related_study_result) + if serial_num is None: + print("Unable to get serial number for study") + related_study_result = get_related_studies( + serial_num, mds_record_guid, hostname + ) + mds_res["gen3_discovery"]["related_studies"] = copy.deepcopy(related_study_result) # merge data from cedar that is not study level metadata into a level higher deleted_keys = [] @@ -324,29 +354,39 @@ def get_related_studies(serial_num, hostname): for key in deleted_keys: del mds_res["gen3_discovery"]["study_metadata"][key] - mds_discovery_data_body = update_filter_metadata(mds_res["gen3_discovery"]) + mds_discovery_data_body = update_filter_metadata( + mds_res["gen3_discovery"] + ) mds_cedar_register_data_body["gen3_discovery"] = mds_discovery_data_body if mds_clinical_trials: - mds_cedar_register_data_body["clinicaltrials_gov"] = {**mds_cedar_register_data_body.get("clinicaltrials_gov", {}), **mds_clinical_trials} + mds_cedar_register_data_body["clinicaltrials_gov"] = { + **mds_cedar_register_data_body.get("clinicaltrials_gov", {}), + **mds_clinical_trials, + } mds_cedar_register_data_body["_guid_type"] = "discovery_metadata" print(f"Metadata {mds_record_guid} is now being registered.") - mds_put = requests.put(f"http://revproxy-service/mds/metadata/{mds_record_guid}", + mds_put = requests.put( + f"http://revproxy-service/mds/metadata/{mds_record_guid}", headers=token_header, - json = mds_cedar_register_data_body + json=mds_cedar_register_data_body, ) if mds_put.status_code == 200: print(f"Successfully registered: {mds_record_guid}") else: - print(f"Failed to register: {mds_record_guid}. Might not be MDS admin") + print( + f"Failed to register: {mds_record_guid}. Might not be MDS admin" + ) print(f"Status from MDS: {mds_put.status_code}") else: print(f"Failed to get information from MDS: {mds.status_code}") - + else: - print(f"Failed to get information from CEDAR wrapper service: {cedar.status_code}") + print( + f"Failed to get information from CEDAR wrapper service: {cedar.status_code}" + ) if offset + limit == total: break From ec6510ff37b03662497ac5e651b36d70f4101e68 Mon Sep 17 00:00:00 2001 From: Jian <52763034+tianj7@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:30:06 -0500 Subject: [PATCH 085/114] add alt text to maintenance page images (#2500) --- files/dashboard/maintenance-page/index.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/files/dashboard/maintenance-page/index.html b/files/dashboard/maintenance-page/index.html index a3e34479b7..fac49e64e1 100644 --- a/files/dashboard/maintenance-page/index.html +++ b/files/dashboard/maintenance-page/index.html @@ -16,7 +16,7 @@
@@ -27,12 +27,12 @@

This site is under maintenance...

Please check back later.

- + A shiba dog looking into the distance
- - + Gen3 Data Commons + Center for Translational Data Science at the University of Chicago
From 5b75af3a39cc8ebcfeed9c75a4fe55f0db004ae1 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Mon, 18 Mar 2024 10:41:44 -0600 Subject: [PATCH 086/114] Update ingress.yaml (#2506) --- kube/services/ingress/ingress.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/ingress/ingress.yaml b/kube/services/ingress/ingress.yaml index 3f1f312592..1db08e8ef6 100644 --- a/kube/services/ingress/ingress.yaml +++ b/kube/services/ingress/ingress.yaml @@ -11,7 +11,7 @@ metadata: alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=600 alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-Res-FIPS-2023-04 + alb.ingress.kubernetes.io/ssl-policy: ELBSecurityPolicy-TLS13-1-2-FIPS-2023-04 spec: ingressClassName: alb rules: From bff3a57818d24f416a3a518bebfe956e281bad80 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 19 Mar 2024 09:53:47 -0500 Subject: [PATCH 087/114] MIDRC-543 Let Hatchery assume role (#2504) --- gen3/bin/kube-setup-hatchery.sh | 31 ++++++++++++++++++++++++-- kube/services/jobs/ecr-access-job.yaml | 2 -- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/gen3/bin/kube-setup-hatchery.sh b/gen3/bin/kube-setup-hatchery.sh index 5454d1e248..bdcff8ed0b 100644 --- a/gen3/bin/kube-setup-hatchery.sh +++ b/gen3/bin/kube-setup-hatchery.sh @@ -76,15 +76,38 @@ else exists_or_create_gen3_license_table "$TARGET_TABLE" fi +# if `nextflow-global.imagebuilder-reader-role-arn` is set in hatchery config, allow hatchery +# to assume the configured role +imagebuilderRoleArn=$(g3kubectl get configmap manifest-hatchery -o jsonpath={.data.nextflow-global} | jq -r '."imagebuilder-reader-role-arn"') +assumeImageBuilderRolePolicyBlock="" +if [ -z "$imagebuilderRoleArn" ]; then + gen3_log_info "No 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, not granting AssumeRole" +else + gen3_log_info "Found 'nexftlow-global.imagebuilder-reader-role-arn' in Hatchery configuration, granting AssumeRole" + assumeImageBuilderRolePolicyBlock=$( cat < /dev/null 2>&1; then roleName="$(gen3 api safe-name hatchery-sa)" gen3 awsrole create $roleName $saName @@ -176,7 +204,6 @@ if ! g3kubectl get sa "$saName" -o json | jq -e '.metadata.annotations | ."eks.a # create the new version gen3_aws_run aws iam create-policy-version --policy-arn "$policyArn" --policy-document "$policy" --set-as-default fi - gen3_log_info "Attaching policy '${policyName}' to role '${roleName}'" gen3 awsrole attach-policy ${policyArn} --role-name ${roleName} --force-aws-cli || exit 1 gen3 awsrole attach-policy "arn:aws:iam::aws:policy/AWSResourceAccessManagerFullAccess" --role-name ${roleName} --force-aws-cli || exit 1 diff --git a/kube/services/jobs/ecr-access-job.yaml b/kube/services/jobs/ecr-access-job.yaml index 11979a1232..89bb49d6d2 100644 --- a/kube/services/jobs/ecr-access-job.yaml +++ b/kube/services/jobs/ecr-access-job.yaml @@ -65,8 +65,6 @@ spec: args: - "-c" - | - set -e - cd cloud-automation/files/scripts/ echo Installing requirements... pip3 install -r ecr-access-job-requirements.txt From 6d67d747679ed73edf55fc3484297732f59e4000 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:10:41 -0400 Subject: [PATCH 088/114] Feat/scaling va workflows (#2507) * Raising total parallelism to 13, to enable stress tests in va-testing. This should be merged with a value of 10, to allow 5 WFs in each env * Bumping the parallelism for Argo up to 5 --- kube/services/argo/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kube/services/argo/values.yaml b/kube/services/argo/values.yaml index 2b46ced0f3..23dda4a5ad 100644 --- a/kube/services/argo/values.yaml +++ b/kube/services/argo/values.yaml @@ -1,6 +1,6 @@ controller: - parallelism: 8 - namespaceParallelism: 3 + parallelism: 10 + namespaceParallelism: 5 metricsConfig: # -- Enables prometheus metrics server enabled: true From 31e6e49015d84782e1c5687175dc11e36bfae4f8 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:41:29 -0400 Subject: [PATCH 089/114] Creating a new type of DB dump that grabs stuff for va-testing (#2508) * Creating a new type of DB dump that grabs stuff for va-testing * Missed changes to dbdump script * Changing job name --- gen3/bin/dbbackup.sh | 13 ++- .../jobs/psql-db-dump-va-testing-job.yaml | 80 +++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 kube/services/jobs/psql-db-dump-va-testing-job.yaml diff --git a/gen3/bin/dbbackup.sh b/gen3/bin/dbbackup.sh index 29f267221d..eb9611a907 100644 --- a/gen3/bin/dbbackup.sh +++ b/gen3/bin/dbbackup.sh @@ -173,6 +173,10 @@ db_restore() { gen3 job run psql-db-prep-restore } +va_testing_db_dump() { + gen3 job run psql-db-dump-va-testing +} + # main function to determine whether dump or restore main() { @@ -191,8 +195,15 @@ main() { create_s3_bucket db_restore ;; + va-dump) + gen3_log_info "Running a va-testing DB dump..." + create_policy + create_service_account_and_role + create_s3_bucket + va_testing_db_dump + ;; *) - echo "Invalid command. Usage: gen3 dbbackup [dump|restore]" + echo "Invalid command. Usage: gen3 dbbackup [dump|restore|va-dump]" return 1 ;; esac diff --git a/kube/services/jobs/psql-db-dump-va-testing-job.yaml b/kube/services/jobs/psql-db-dump-va-testing-job.yaml new file mode 100644 index 0000000000..8a8037e166 --- /dev/null +++ b/kube/services/jobs/psql-db-dump-va-testing-job.yaml @@ -0,0 +1,80 @@ +--- +# NOTE: This job was created specifically to dump all the databases in va-testing, in preparation for a move to second cluster +# If you aren't doing that, this probably is not the job you're looking for +apiVersion: batch/v1 +kind: Job +metadata: + name: psql-db-dump-va-testing +spec: + template: + metadata: + labels: + app: gen3job + spec: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - weight: 99 + preference: + matchExpressions: + - key: eks.amazonaws.com/capacityType + operator: In + values: + - ONDEMAND + serviceAccountName: dbbackup-sa + containers: + - name: pgdump + image: quay.io/cdis/awshelper:master + imagePullPolicy: Always + env: + - name: gen3Env + valueFrom: + configMapKeyRef: + name: global + key: environment + - name: JENKINS_HOME + value: "devterm" + - name: GEN3_HOME + value: /home/ubuntu/cloud-automation + command: ["/bin/bash"] + args: + - "-c" + - | + source "${GEN3_HOME}/gen3/lib/utils.sh" + gen3_load "gen3/gen3setup" + account_id=$(aws sts get-caller-identity --query "Account" --output text) + default_bucket_name="gen3-db-backups-${account_id}" + default_databases=("fence" "indexd" "sheepdog" "peregrine" "arborist" "argo" "atlas" "metadata" "ohdsi" "omop-data" "wts") + s3_dir="va-testing-$(date +"%Y-%m-%d-%H-%M-%S")" + databases=("${default_databases[@]}") + bucket_name=$default_bucket_name + + for database in "${databases[@]}"; do + gen3_log_info "Starting database backup for ${database}" + gen3 db backup "${database}" > "${database}.sql" + + if [ $? -eq 0 ] && [ -f "${database}.sql" ]; then + gen3_log_info "Uploading backup file ${database}.sql to s3://${bucket_name}/${s3_dir}/${database}.sql" + aws s3 cp "${database}.sql" "s3://${bucket_name}/${s3_dir}/${database}.sql" + + if [ $? -eq 0 ]; then + gen3_log_info "Successfully uploaded ${database}.sql to S3" + else + gen3_log_err "Failed to upload ${database}.sql to S3" + fi + gen3_log_info "Deleting temporary backup file ${database}.sql" + rm -f "${database}.sql" + else + gen3_log_err "Backup operation failed for ${database}" + rm -f "${database}.sql" + fi + done + sleep 600 + restartPolicy: Never From 6de65e70a7065789f6250ad05e94f816bf8eeeaf Mon Sep 17 00:00:00 2001 From: Michael Lukowski Date: Wed, 20 Mar 2024 15:32:38 -0500 Subject: [PATCH 090/114] add whitelist for qdr staging (#2509) --- files/squid_whitelist/web_whitelist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index c191b2e8c1..afacba9e4c 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -165,3 +165,5 @@ www.rabbitmq.com www.uniprot.org vpodc.org yahoo.com +idp.stage.qdr.org +stage.qdr.org \ No newline at end of file From cde8a9666a53fe9c2345f4562f202a701e9a172a Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:26:58 -0400 Subject: [PATCH 091/114] Cronjobs aren't beta (#2511) --- .../services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml b/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml index 74d7fc9a4d..93eaf7652d 100644 --- a/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml +++ b/kube/services/jobs/fence-cleanup-expired-ga4gh-info-cronjob.yaml @@ -1,5 +1,5 @@ --- -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: fence-cleanup-expired-ga4gh-info From 79f305a7741a221a6a6e0236c08e0f610cc589f0 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:13:38 -0500 Subject: [PATCH 092/114] MIDRC-672 Fix ECR access job role name conflict (#2515) --- gen3/bin/iam-serviceaccount.sh | 25 +++++++++++++++-------- gen3/bin/kube-setup-ecr-access-cronjob.sh | 4 ++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/gen3/bin/iam-serviceaccount.sh b/gen3/bin/iam-serviceaccount.sh index 0c5a8bba3c..1ea055f662 100644 --- a/gen3/bin/iam-serviceaccount.sh +++ b/gen3/bin/iam-serviceaccount.sh @@ -115,7 +115,7 @@ EOF # @return the resulting json from awscli ## function create_role(){ - local role_name="${vpc_name}-${SERVICE_ACCOUNT_NAME}-role" + local role_name="${1}" if [[ ${#role_name} -gt 63 ]]; then role_name=$(echo "$role_name" | head -c63) gen3_log_warning "Role name has been truncated, due to amazon role name 64 character limit. New role name is $role_name" @@ -123,8 +123,8 @@ function create_role(){ local assume_role_policy_path="$(create_assume_role_policy)" gen3_log_info "Entering create_role" - gen3_log_info " ${role_name}" - gen3_log_info " ${assume_role_policy_path}" + gen3_log_info " Role: ${role_name}" + gen3_log_info " Policy path: ${assume_role_policy_path}" local role_json role_json=$(aws iam create-role \ @@ -156,8 +156,8 @@ function add_policy_to_role(){ local role_name="${2}" gen3_log_info "Entering add_policy_to_role" - gen3_log_info " ${policy}" - gen3_log_info " ${role_name}" + gen3_log_info " Policy: ${policy}" + gen3_log_info " Role: ${role_name}" local result if [[ ${policy} =~ arn:aws:iam::aws:policy/[a-zA-Z0-9]+ ]] @@ -198,8 +198,8 @@ function create_role_with_policy() { local role_name="${2}" gen3_log_info "Entering create_role_with_policy" - gen3_log_info " ${policy}" - gen3_log_info " ${role_name}" + gen3_log_info " Policy: ${policy}" + gen3_log_info " Role: ${role_name}" local created_role_json created_role_json="$(create_role ${role_name})" || return $? @@ -357,7 +357,10 @@ function main() { local policy_validation local policy_source - local role_name="${vpc_name}-${SERVICE_ACCOUNT_NAME}-role" + local role_name=$ROLE_NAME + if [ -z "${role_name}" ]; then + role_name="${vpc_name}-${SERVICE_ACCOUNT_NAME}-role" + fi if [ -z ${NAMESPACE_SCRIPT} ]; then @@ -481,6 +484,12 @@ while getopts "$OPTSPEC" optchar; do ACTION="c" SERVICE_ACCOUNT_NAME=${OPTARG#*=} ;; + role-name) + ROLE_NAME="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) + ;; + role-name=*) + ROLE_NAME=${OPTARG#*=} + ;; list) ACTION="l" SERVICE_ACCOUNT_NAME="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 )) diff --git a/gen3/bin/kube-setup-ecr-access-cronjob.sh b/gen3/bin/kube-setup-ecr-access-cronjob.sh index d23afc862c..5c645ad35d 100644 --- a/gen3/bin/kube-setup-ecr-access-cronjob.sh +++ b/gen3/bin/kube-setup-ecr-access-cronjob.sh @@ -38,8 +38,8 @@ setup_ecr_access_job() { ] } EOM - local role_name - if ! role_name="$(gen3 iam-serviceaccount -c "${saName}" -p $tempFile)" || [[ -z "$role_name" ]]; then + local safe_role_name=$(gen3 api safe-name ${saName}-role | head -c63) + if ! role_name="$(gen3 iam-serviceaccount -c "${saName}" -p $tempFile --role-name $safe_role_name)" || [[ -z "$role_name" ]]; then gen3_log_err "Failed to create iam service account" rm $tempFile return 1 From 797fdf3fcd4f2ce8d66582a6e7891bfbba5bffe1 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Fri, 29 Mar 2024 11:59:34 -0400 Subject: [PATCH 093/114] Adding awslabs.github.io to the squid whitelist (#2516) --- files/squid_whitelist/web_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/files/squid_whitelist/web_whitelist b/files/squid_whitelist/web_whitelist index afacba9e4c..6896314abb 100644 --- a/files/squid_whitelist/web_whitelist +++ b/files/squid_whitelist/web_whitelist @@ -7,6 +7,7 @@ achecker.ca apache.github.io api.epigraphdb.org api.monqcle.com +awslabs.github.io biodata-integration-tests.net marketing.biorender.com clinicaltrials.gov From 07813b6fff305398b90c6dcdc810e253fb41b086 Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Thu, 4 Apr 2024 10:37:08 -0500 Subject: [PATCH 094/114] Fail if fence-create client fails in kube-setup-ohdsi (#2514) Co-authored-by: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> --- gen3/bin/kube-setup-ohdsi.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/gen3/bin/kube-setup-ohdsi.sh b/gen3/bin/kube-setup-ohdsi.sh index 14b35a7146..3d8165547f 100644 --- a/gen3/bin/kube-setup-ohdsi.sh +++ b/gen3/bin/kube-setup-ohdsi.sh @@ -14,13 +14,8 @@ new_client() { local secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client atlas --urls https://${atlas_hostname}/WebAPI/user/oauth/callback?client_name=OidcClient --username atlas --allowed-scopes openid profile email user | tail -1) # secrets looks like ('CLIENT_ID', 'CLIENT_SECRET') if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then - # try delete client - g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-delete --client atlas > /dev/null 2>&1 - secrets=$(g3kubectl exec -c fence $(gen3 pod fence) -- fence-create client-create --client atlas --urls https://${atlas_hostname}/WebAPI/user/oauth/callback?client_name=OidcClient --username atlas --allowed-scopes openid profile email user | tail -1) - if [[ ! $secrets =~ (\'(.*)\', \'(.*)\') ]]; then - gen3_log_err "kube-setup-ohdsi" "Failed generating oidc client for atlas: $secrets" - return 1 - fi + gen3_log_err "kube-setup-ohdsi" "Failed generating oidc client for atlas: $secrets" + return 1 fi local FENCE_CLIENT_ID="${BASH_REMATCH[2]}" local FENCE_CLIENT_SECRET="${BASH_REMATCH[3]}" From 775d224e4ffc301e04a78c7878b499caf05d1f0f Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:05:45 -0500 Subject: [PATCH 095/114] Update ecr.sh (#2518) --- gen3/bin/ecr.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gen3/bin/ecr.sh b/gen3/bin/ecr.sh index 930202a876..f3f13b993a 100644 --- a/gen3/bin/ecr.sh +++ b/gen3/bin/ecr.sh @@ -32,6 +32,7 @@ accountList=( 205252583234 885078588865 922467707295 +533267425233 ) principalStr="" From 9c2f09eefbcdc1cc29286481cb0bafac170a4bf1 Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Thu, 4 Apr 2024 13:16:43 -0500 Subject: [PATCH 096/114] Update ecr.sh (#2519) --- gen3/bin/ecr.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gen3/bin/ecr.sh b/gen3/bin/ecr.sh index f3f13b993a..36af791ef6 100644 --- a/gen3/bin/ecr.sh +++ b/gen3/bin/ecr.sh @@ -33,6 +33,7 @@ accountList=( 885078588865 922467707295 533267425233 +048463324059 ) principalStr="" From 147ea5e0086a0753536a0f2f027471544b638bea Mon Sep 17 00:00:00 2001 From: emalinowski Date: Mon, 15 Apr 2024 06:14:13 -0600 Subject: [PATCH 097/114] fix(sqs-helper): Updated sqs helper script to create more than one workspace GPE-998 (#2295) * fix(sqs-helper): Updated sqs helper script to create more than one workspace * fix(sqs-helper): Updated scripts calling sqs helper to give simpler sqs names * fix(sqs-helper): Updated sqs variables to be consistent --------- Co-authored-by: Edward Malinowski --- gen3/bin/kube-setup-audit-service.sh | 2 +- gen3/bin/kube-setup-fence.sh | 2 +- gen3/bin/kube-setup-karpenter.sh | 6 +++--- gen3/bin/sqs.sh | 15 ++++++++------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/gen3/bin/kube-setup-audit-service.sh b/gen3/bin/kube-setup-audit-service.sh index b7565194cd..92c70f352c 100644 --- a/gen3/bin/kube-setup-audit-service.sh +++ b/gen3/bin/kube-setup-audit-service.sh @@ -65,7 +65,7 @@ EOM } setup_audit_sqs() { - local sqsName="$(gen3 api safe-name audit-sqs)" + local sqsName="audit-sqs" sqsInfo="$(gen3 sqs create-queue-if-not-exist $sqsName)" || exit 1 sqsUrl="$(jq -e -r '.["url"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; } sqsArn="$(jq -e -r '.["arn"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; } diff --git a/gen3/bin/kube-setup-fence.sh b/gen3/bin/kube-setup-fence.sh index 03edabbf4c..cc0516c93a 100644 --- a/gen3/bin/kube-setup-fence.sh +++ b/gen3/bin/kube-setup-fence.sh @@ -9,7 +9,7 @@ source "${GEN3_HOME}/gen3/lib/utils.sh" gen3_load "gen3/lib/kube-setup-init" setup_audit_sqs() { - local sqsName="$(gen3 api safe-name audit-sqs)" + local sqsName="audit-sqs" sqsInfo="$(gen3 sqs create-queue-if-not-exist $sqsName)" || exit 1 sqsUrl="$(jq -e -r '.["url"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; } sqsArn="$(jq -e -r '.["arn"]' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; } diff --git a/gen3/bin/kube-setup-karpenter.sh b/gen3/bin/kube-setup-karpenter.sh index 2737ed6eeb..949c1ccd13 100644 --- a/gen3/bin/kube-setup-karpenter.sh +++ b/gen3/bin/kube-setup-karpenter.sh @@ -31,7 +31,7 @@ gen3_deploy_karpenter() { else karpenter=${karpenter:-v0.22.0} fi - local queue_name="karpenter-sqs-${vpc_name}" + local queue_name="$(gen3 api safe-name karpenter-sqs)" echo '{ "Statement": [ { @@ -202,9 +202,9 @@ gen3_update_karpenter_configs() { } gen3_create_karpenter_sqs_eventbridge() { - local queue_name="karpenter-sqs-${vpc_name}" + local queue_name="$(gen3 api safe-name karpenter-sqs)" local eventbridge_rule_name="karpenter-eventbridge-${vpc_name}" - #gen3 sqs create-queue-if-not-exist $queue_name >> "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json" + gen3 sqs create-queue-if-not-exist karpenter-sqs >> "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json" local queue_url=$(cat "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json" | jq -r '.url') local queue_arn=$(cat "$XDG_RUNTIME_DIR/sqs-${vpc_name}.json" | jq -r '.arn') # Create eventbridge rules diff --git a/gen3/bin/sqs.sh b/gen3/bin/sqs.sh index dccb1ff7b7..7448437a0c 100644 --- a/gen3/bin/sqs.sh +++ b/gen3/bin/sqs.sh @@ -50,15 +50,15 @@ EOM # @sqsName # gen3_sqs_create_queue() { - local sqsName=$1 - if ! shift || [[ -z "$sqsName" ]]; then - gen3_log_err "Must provide 'sqsName' to 'gen3_sqs_create_queue'" + local serviceName=$1 + if ! shift || [[ -z "$serviceName" ]]; then + gen3_log_err "Must provide 'serviceName' to 'gen3_sqs_create_queue'" return 1 fi + local sqsName="$(gen3 api safe-name $serviceName)" gen3_log_info "Creating SQS '$sqsName'" - local prefix="$(gen3 api safe-name sqs-create)" ( # subshell - do not pollute parent environment - gen3 workon default ${prefix}__sqs 1>&2 + gen3 workon default ${sqsName}__sqs 1>&2 gen3 cd 1>&2 cat << EOF > config.tfvars sqs_name="$sqsName" @@ -76,7 +76,8 @@ EOF # @sqsName # gen3_sqs_create_queue_if_not_exist() { - local sqsName=$1 + local serviceName=$1 + local sqsName="$(gen3 api safe-name $serviceName)" if ! shift || [[ -z "$sqsName" ]]; then gen3_log_err "Must provide 'sqsName' to 'gen3_sqs_create_queue'" return 1 @@ -90,7 +91,7 @@ gen3_sqs_create_queue_if_not_exist() { gen3_log_info "The '$sqsName' SQS already exists" else # create the queue - sqsInfo="$(gen3_sqs_create_queue $sqsName)" || exit 1 + sqsInfo="$(gen3_sqs_create_queue $serviceName)" || exit 1 sqsUrl="$(jq -e -r '.["sqs-url"].value' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-url' from output: $sqsInfo"; exit 1; } sqsArn="$(jq -e -r '.["sqs-arn"].value' <<< "$sqsInfo")" || { echo "Cannot get 'sqs-arn' from output: $sqsInfo"; exit 1; } fi From d55a3862609339149ac9373d708aac6546267618 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:34:51 -0600 Subject: [PATCH 098/114] Update web_wildcard_whitelist (#2523) --- files/squid_whitelist/web_wildcard_whitelist | 1 + 1 file changed, 1 insertion(+) diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist index b71ee76c20..1374c5d679 100644 --- a/files/squid_whitelist/web_wildcard_whitelist +++ b/files/squid_whitelist/web_wildcard_whitelist @@ -11,6 +11,7 @@ .bioconductor.org .bionimbus.org .bitbucket.org +.blob.core.windows.net .bloodpac.org .braincommons.org .bsc.es From 9056d2b7754aeaec60628e5238958ae309494771 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 19 Apr 2024 10:56:16 -0600 Subject: [PATCH 099/114] adding a cron to check qaplanetv1 for the fenceshib service since it can break revproxy/automation (#2525) --- .../node-monitors/fenceshib-jenkins-test.yaml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 kube/services/node-monitors/fenceshib-jenkins-test.yaml diff --git a/kube/services/node-monitors/fenceshib-jenkins-test.yaml b/kube/services/node-monitors/fenceshib-jenkins-test.yaml new file mode 100644 index 0000000000..e9e27af983 --- /dev/null +++ b/kube/services/node-monitors/fenceshib-jenkins-test.yaml @@ -0,0 +1,40 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: fenceshib-service-check + namespace: default +spec: + schedule: "0 */4 * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: gen3job + spec: + serviceAccountName: node-monitor + containers: + - name: kubectl + image: quay.io/cdis/awshelper + env: + - name: SLACK_WEBHOOK_URL + valueFrom: + configMapKeyRef: + name: global + key: slack_webhook + command: ["/bin/bash"] + args: + - "-c" + - | + #!/bin/bash + + fenceshib=$(kubectl get services -A | grep "fenceshib-service" | awk '{print $2}') + + # Check if there are any fenceshib services + if [[ ! -z "$fenceshib" ]]; then + echo "Alert: Service fenceshib-service found with output: $fenceshib" + curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"WARNING: Fenceshib service discovered in qaplanetv1 cluster. This could cause issues with future CI runs. Please delete this service if it is not needed. Run the following in qaplanetv1 to see which namespace it is in: \`kubectl get services -A | grep "fenceshib-service"\`\"}" $SLACK_WEBHOOK_URL + else + echo "Fenceshib Service Not Found" + fi + restartPolicy: OnFailure From 5dd762ee9389b3284872bf66ba46739ca6aaf1c7 Mon Sep 17 00:00:00 2001 From: Andrew Prokhorenkov Date: Fri, 19 Apr 2024 12:35:16 -0500 Subject: [PATCH 100/114] feat: update instance types with newer generation (#2527) --- .../argo-events/workflows/configmap.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/kube/services/argo-events/workflows/configmap.yaml b/kube/services/argo-events/workflows/configmap.yaml index ae1c16653c..8d90457145 100644 --- a/kube/services/argo-events/workflows/configmap.yaml +++ b/kube/services/argo-events/workflows/configmap.yaml @@ -27,22 +27,48 @@ data: - c6a.4xlarge - c6a.8xlarge - c6a.12xlarge + - c7a.large + - c7a.xlarge + - c7a.2xlarge + - c7a.4xlarge + - c7a.8xlarge + - c7a.12xlarge - c6i.large - c6i.xlarge - c6i.2xlarge - c6i.4xlarge - c6i.8xlarge - c6i.12xlarge + - c7i.large + - c7i.xlarge + - c7i.2xlarge + - c7i.4xlarge + - c7i.8xlarge + - c7i.12xlarge - m6a.2xlarge - m6a.4xlarge - m6a.8xlarge - m6a.12xlarge - m6a.16xlarge + - m6a.24xlarge + - m7a.2xlarge + - m7a.4xlarge + - m7a.8xlarge + - m7a.12xlarge + - m7a.16xlarge + - m7a.24xlarge - m6i.2xlarge - m6i.4xlarge - m6i.8xlarge - m6i.12xlarge - m6i.16xlarge + - m6i.24xlarge + - m7i.2xlarge + - m7i.4xlarge + - m7i.8xlarge + - m7i.12xlarge + - m7i.16xlarge + - m7i.24xlarge taints: - key: role value: WORKFLOW_NAME From b68908ffb6b81e2300843175b14d4efeb0fbc5a4 Mon Sep 17 00:00:00 2001 From: emalinowski Date: Tue, 23 Apr 2024 10:07:47 -0600 Subject: [PATCH 101/114] fix(cronjob-apis): Updated decprecated cronjob apis (#2529) Co-authored-by: Edward Malinowski --- kube/services/jobs/arborist-rm-expired-access-cronjob.yaml | 2 +- kube/services/jobs/covid19-bayes-cronjob.yaml | 2 +- kube/services/jobs/etl-cronjob.yaml | 2 +- kube/services/jobs/fence-visa-update-cronjob.yaml | 2 +- kube/services/jobs/google-delete-expired-access-cronjob.yaml | 2 +- .../jobs/google-delete-expired-service-account-cronjob.yaml | 4 ++-- kube/services/jobs/google-init-proxy-groups-cronjob.yaml | 4 ++-- kube/services/jobs/google-manage-account-access-cronjob.yaml | 4 ++-- kube/services/jobs/google-manage-keys-cronjob.yaml | 4 ++-- .../jobs/google-verify-bucket-access-group-cronjob.yaml | 4 ++-- kube/services/jobs/healthcheck-cronjob.yaml | 2 +- kube/services/jobs/s3sync-cronjob.yaml | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/kube/services/jobs/arborist-rm-expired-access-cronjob.yaml b/kube/services/jobs/arborist-rm-expired-access-cronjob.yaml index 29603d27fe..a726237362 100644 --- a/kube/services/jobs/arborist-rm-expired-access-cronjob.yaml +++ b/kube/services/jobs/arborist-rm-expired-access-cronjob.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: arborist-rm-expired-access diff --git a/kube/services/jobs/covid19-bayes-cronjob.yaml b/kube/services/jobs/covid19-bayes-cronjob.yaml index 733c17cf71..01e71badeb 100644 --- a/kube/services/jobs/covid19-bayes-cronjob.yaml +++ b/kube/services/jobs/covid19-bayes-cronjob.yaml @@ -1,5 +1,5 @@ # gen3 job run covid19-bayes-cronjob S3_BUCKET -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: covid19-bayes diff --git a/kube/services/jobs/etl-cronjob.yaml b/kube/services/jobs/etl-cronjob.yaml index 463fbfb2e2..95b423debd 100644 --- a/kube/services/jobs/etl-cronjob.yaml +++ b/kube/services/jobs/etl-cronjob.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: etl diff --git a/kube/services/jobs/fence-visa-update-cronjob.yaml b/kube/services/jobs/fence-visa-update-cronjob.yaml index 6c58ef291e..eba842ddf5 100644 --- a/kube/services/jobs/fence-visa-update-cronjob.yaml +++ b/kube/services/jobs/fence-visa-update-cronjob.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: fence-visa-update diff --git a/kube/services/jobs/google-delete-expired-access-cronjob.yaml b/kube/services/jobs/google-delete-expired-access-cronjob.yaml index ce485cce36..2b9e4e49a6 100644 --- a/kube/services/jobs/google-delete-expired-access-cronjob.yaml +++ b/kube/services/jobs/google-delete-expired-access-cronjob.yaml @@ -1,5 +1,5 @@ --- -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: google-delete-expired-access diff --git a/kube/services/jobs/google-delete-expired-service-account-cronjob.yaml b/kube/services/jobs/google-delete-expired-service-account-cronjob.yaml index eb102f5bf4..b40e22624d 100644 --- a/kube/services/jobs/google-delete-expired-service-account-cronjob.yaml +++ b/kube/services/jobs/google-delete-expired-service-account-cronjob.yaml @@ -1,6 +1,6 @@ --- -# Note: change to batch/v1beta1 once we bump to k8s 1.8 -apiVersion: batch/v1beta1 +# Note: change to batch/v1 once we bump to k8s 1.8 +apiVersion: batch/v1 kind: CronJob metadata: name: google-delete-expired-service-account diff --git a/kube/services/jobs/google-init-proxy-groups-cronjob.yaml b/kube/services/jobs/google-init-proxy-groups-cronjob.yaml index 499d6cabd1..6b4fc10aa0 100644 --- a/kube/services/jobs/google-init-proxy-groups-cronjob.yaml +++ b/kube/services/jobs/google-init-proxy-groups-cronjob.yaml @@ -1,6 +1,6 @@ --- -# Note: change to batch/v1beta1 once we bump to k8s 1.8 -apiVersion: batch/v1beta1 +# Note: change to batch/v1 once we bump to k8s 1.8 +apiVersion: batch/v1 kind: CronJob metadata: name: google-init-proxy-groups diff --git a/kube/services/jobs/google-manage-account-access-cronjob.yaml b/kube/services/jobs/google-manage-account-access-cronjob.yaml index 4e796cea0d..fd8bba6067 100644 --- a/kube/services/jobs/google-manage-account-access-cronjob.yaml +++ b/kube/services/jobs/google-manage-account-access-cronjob.yaml @@ -1,6 +1,6 @@ --- -# Note: change to batch/v1beta1 once we bump to k8s 1.8 -apiVersion: batch/v1beta1 +# Note: change to batch/v1 once we bump to k8s 1.8 +apiVersion: batch/v1 kind: CronJob metadata: name: google-manage-account-access diff --git a/kube/services/jobs/google-manage-keys-cronjob.yaml b/kube/services/jobs/google-manage-keys-cronjob.yaml index ea0bcc45fd..eff76d30ad 100644 --- a/kube/services/jobs/google-manage-keys-cronjob.yaml +++ b/kube/services/jobs/google-manage-keys-cronjob.yaml @@ -1,6 +1,6 @@ --- -# Note: change to batch/v1beta1 once we bump to k8s 1.8 -apiVersion: batch/v1beta1 +# Note: change to batch/v1 once we bump to k8s 1.8 +apiVersion: batch/v1 kind: CronJob metadata: name: google-manage-keys diff --git a/kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml b/kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml index 57981d813f..49e83374fc 100644 --- a/kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml +++ b/kube/services/jobs/google-verify-bucket-access-group-cronjob.yaml @@ -1,6 +1,6 @@ --- -# Note: change to batch/v1beta1 once we bump to k8s 1.8 -apiVersion: batch/v1beta1 +# Note: change to batch/v1 once we bump to k8s 1.8 +apiVersion: batch/v1 kind: CronJob metadata: name: google-verify-bucket-access-group diff --git a/kube/services/jobs/healthcheck-cronjob.yaml b/kube/services/jobs/healthcheck-cronjob.yaml index d79274bb7f..1ca71fc8d6 100644 --- a/kube/services/jobs/healthcheck-cronjob.yaml +++ b/kube/services/jobs/healthcheck-cronjob.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: healthcheck diff --git a/kube/services/jobs/s3sync-cronjob.yaml b/kube/services/jobs/s3sync-cronjob.yaml index f05ab518a4..69d66ec3fa 100644 --- a/kube/services/jobs/s3sync-cronjob.yaml +++ b/kube/services/jobs/s3sync-cronjob.yaml @@ -5,7 +5,7 @@ #####REQUIRED VARIABLE######## #SOURCE_BUCKET #TARGET_BUCKET -apiVersion: batch/v1beta1 +apiVersion: batch/v1 kind: CronJob metadata: name: s3sync From 40be00d3460af4c133ea0c4bcb675620be56dfce Mon Sep 17 00:00:00 2001 From: Mingfei Shao <2475897+mfshao@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:17:55 -0500 Subject: [PATCH 102/114] fix: skip setting up cedar-ingest client cred in ci (#2526) * fix: skip setting up cedar-ingest client cred in ci * fix cleanup --------- Co-authored-by: Hara Prasad --- gen3/bin/kube-setup-cedar-wrapper.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gen3/bin/kube-setup-cedar-wrapper.sh b/gen3/bin/kube-setup-cedar-wrapper.sh index c8f0d03c6c..a56bebc406 100644 --- a/gen3/bin/kube-setup-cedar-wrapper.sh +++ b/gen3/bin/kube-setup-cedar-wrapper.sh @@ -60,8 +60,12 @@ if ! g3kubectl get secrets/cedar-g3auto > /dev/null 2>&1; then return 1 fi -gen3_log_info "Checking cedar-client creds" -setup_creds +if [[ -n "$JENKINS_HOME" ]]; then + gen3_log_info "Skipping cedar-client creds setup in non-adminvm environment" +else + gen3_log_info "Checking cedar-client creds" + setup_creds +fi if ! gen3 secrets decode cedar-g3auto cedar_api_key.txt > /dev/null 2>&1; then gen3_log_err "No CEDAR api key present in cedar-g3auto secret, not rolling CEDAR wrapper" From ec4053eb7eb41a12b0aa65aab96343ae03d8fff3 Mon Sep 17 00:00:00 2001 From: Luca Graglia Date: Tue, 30 Apr 2024 14:13:48 -0500 Subject: [PATCH 103/114] Update workon.sh (#2039) Co-authored-by: jawadqur <55899496+jawadqur@users.noreply.github.com> --- gen3/bin/workon.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gen3/bin/workon.sh b/gen3/bin/workon.sh index e7b951d1ca..f614cf662d 100644 --- a/gen3/bin/workon.sh +++ b/gen3/bin/workon.sh @@ -113,7 +113,7 @@ if [[ ! -f "$bucketCheckFlag" && "$GEN3_FLAVOR" == "AWS" ]]; then } EOM ) - gen3_aws_run aws s3api create-bucket --acl private --bucket "$GEN3_S3_BUCKET" --create-bucket-configuration ‘{“LocationConstraint”:“‘$(aws configure get $GEN3_PROFILE.region)‘“}’ + gen3_aws_run aws s3api create-bucket --acl private --bucket "$GEN3_S3_BUCKET" $([[ $(aws configure get $GEN3_PROFILE.region) = "us-east-1" ]] && echo "" || echo --create-bucket-configuration LocationConstraint="$(aws configure get $GEN3_PROFILE.region)") sleep 5 # Avoid race conditions if gen3_aws_run aws s3api put-bucket-encryption --bucket "$GEN3_S3_BUCKET" --server-side-encryption-configuration "$S3_POLICY"; then touch "$bucketCheckFlag" From 7f56512a802a7ee6579359fe9cbc7f033300376a Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 1 May 2024 14:54:44 -0400 Subject: [PATCH 104/114] Fix/karpenter setup with sed (#2536) * Fixing the sed command in the create karpenter resources job * Fixing some silliness * Please let me blame that on Friday brain * Let's do it * What was I doing before? * Using a more robust method for grabbing workflow and usernames --- .../argo-events/workflows/sensor-created.yaml | 20 ++++++++++++++--- .../karpenter-reconciler-cronjob.yaml | 22 ++++++++++--------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/kube/services/argo-events/workflows/sensor-created.yaml b/kube/services/argo-events/workflows/sensor-created.yaml index 4221f57423..9f6de2c832 100644 --- a/kube/services/argo-events/workflows/sensor-created.yaml +++ b/kube/services/argo-events/workflows/sensor-created.yaml @@ -59,12 +59,22 @@ spec: args: - "-c" - | + #!/bin/bash + if [ -z "$PROVISIONER_TEMPLATE" ]; then + PROVISIONER_TEMPLATE="provisioner.yaml" + fi + + if [ -z "$AWSNODETEMPLATE_TEMPLATE" ]; then + AWSNODETEMPLATE_TEMPLATE="nodetemplate.yaml" + fi + + if ! kubectl get awsnodetemplate workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" | kubectl apply -f - + sed -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f - fi if ! kubectl get provisioner workflow-$WORKFLOW_NAME >/dev/null 2>&1; then - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" | kubectl apply -f - + sed -e "s/WORKFLOW_NAME/$WORKFLOW_NAME/" -e "s/GEN3_USERNAME/$GEN3_USERNAME/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f - fi env: - name: WORKFLOW_NAME @@ -76,9 +86,13 @@ spec: configMapKeyRef: name: environment key: environment + - name: PROVISIONER_TEMPLATE + value: /manifests/provisioner.yaml + - name: AWSNODETEMPLATE_TEMPLATE + value: /manifests/nodetemplate.yaml volumeMounts: - name: karpenter-templates-volume - mountPath: /home/manifests + mountPath: /manifests volumes: - name: karpenter-templates-volume configMap: diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml index 4f82e9d43e..aef5d6c49f 100644 --- a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob.yaml @@ -43,9 +43,7 @@ spec: ENVIRONMENT=$(kubectl -n default get configmap global -o jsonpath="{.data.environment}") - RAW_WORKFLOWS=$(kubectl get workflows -n argo -o yaml) - - WORKFLOWS=$(echo "${RAW_WORKFLOWS}" | yq -r '.items[] | [.metadata.name, .metadata.labels.gen3username] | join(" ")') + WORKFLOWS=$(kubectl get workflows -n argo -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.gen3username}{"\n"}') WORKFLOW_ARRAY=() @@ -53,20 +51,24 @@ spec: WORKFLOW_ARRAY+=("$line") done <<< "$WORKFLOWS" + echo $WORKFLOWS + for workflow in "${WORKFLOW_ARRAY[@]}" do workflow_name=$(echo "$workflow" | awk '{print $1}') workflow_user=$(echo "$workflow" | awk '{print $2}') - if ! kubectl get awsnodetemplate workflow-$workflow_name >/dev/null 2>&1; then - echo "No awsnodetemplate found for ${workflow_name}, creating one" - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f - - fi + if [ ! -z "$workflow_name" ]; then + if ! kubectl get awsnodetemplate workflow-$workflow_name >/dev/null 2>&1; then + echo "No awsnodetemplate found for ${workflow_name}, creating one" + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f - + fi - if ! kubectl get provisioner workflow-$workflow_name >/dev/null 2>&1; then - echo "No provisioner found for ${workflow_name}, creating one" - sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f - + if ! kubectl get provisioner workflow-$workflow_name >/dev/null 2>&1; then + echo "No provisioner found for ${workflow_name}, creating one" + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f - + fi fi done restartPolicy: OnFailure From 4d346fffe9631ebe31d0297d17a885cc7f598867 Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Wed, 1 May 2024 16:15:07 -0400 Subject: [PATCH 105/114] Let's get the va-testing reconciler onto master (#2537) --- ...rpenter-reconciler-cronjob-va-testing.yaml | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 kube/services/karpenter-reconciler/karpenter-reconciler-cronjob-va-testing.yaml diff --git a/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob-va-testing.yaml b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob-va-testing.yaml new file mode 100644 index 0000000000..aaba57b07a --- /dev/null +++ b/kube/services/karpenter-reconciler/karpenter-reconciler-cronjob-va-testing.yaml @@ -0,0 +1,71 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: karpenter-reconciler-cronjob-va-testing + namespace: argo-events +spec: + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: gen3job + spec: + serviceAccount: karpenter-reconciler + volumes: + - name: karpenter-templates-volume + configMap: + name: karpenter-templates + containers: + - name: karpenter-reconciler + image: quay.io/cdis/awshelper + volumeMounts: + - name: karpenter-templates-volume + mountPath: /manifests + env: + - name: PROVISIONER_TEMPLATE + value: /manifests/provisioner.yaml + - name: AWSNODETEMPLATE_TEMPLATE + value: /manifests/nodetemplate.yaml + command: ["/bin/bash"] + args: + - "-c" + - | + #!/bin/bash + if [ -z "$PROVISIONER_TEMPLATE" ]; then + PROVISIONER_TEMPLATE="provisioner.yaml" + fi + + if [ -z "$AWSNODETEMPLATE_TEMPLATE" ]; then + AWSNODETEMPLATE_TEMPLATE="nodetemplate.yaml" + fi + + ENVIRONMENT=$(kubectl -n va-testing get configmap global -o jsonpath="{.data.environment}") + + WORKFLOWS=$(kubectl get workflows -n argo -o=jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.gen3username}{"\n"}') + + WORKFLOW_ARRAY=() + + while IFS= read -r line; do + WORKFLOW_ARRAY+=("$line") + done <<< "$WORKFLOWS" + + for workflow in "${WORKFLOW_ARRAY[@]}" + do + echo "Running loop for workflow: $workflow" + workflow_name=$(echo "$workflow" | awk '{print $1}') + workflow_user=$(echo "$workflow" | awk '{print $2}') + + if ! kubectl get awsnodetemplate workflow-$workflow_name >/dev/null 2>&1; then + echo "No awsnodetemplate found for ${workflow_name}, creating one" + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$AWSNODETEMPLATE_TEMPLATE" | kubectl apply -f - + fi + + if ! kubectl get provisioner workflow-$workflow_name >/dev/null 2>&1; then + echo "No provisioner found for ${workflow_name}, creating one" + sed -e "s/WORKFLOW_NAME/$workflow_name/" -e "s/GEN3_USERNAME/$workflow_user/" -e "s/ENVIRONMENT/$ENVIRONMENT/" "$PROVISIONER_TEMPLATE" | kubectl apply -f - + + fi + done + restartPolicy: OnFailure From 76831781e04480dc78279b56bfd077980e7728cf Mon Sep 17 00:00:00 2001 From: Hara Prasad Date: Wed, 1 May 2024 14:13:08 -0700 Subject: [PATCH 106/114] Roll arborist before indexd (#2535) --- gen3/bin/kube-roll-all.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gen3/bin/kube-roll-all.sh b/gen3/bin/kube-roll-all.sh index 1dca87c68c..744e8e2889 100644 --- a/gen3/bin/kube-roll-all.sh +++ b/gen3/bin/kube-roll-all.sh @@ -51,20 +51,20 @@ fi gen3 kube-setup-networkpolicy disable # -# Hopefull core secrets/config in place - start bringing up services +# Hopefully core secrets/config in place - start bringing up services # -if g3k_manifest_lookup .versions.indexd 2> /dev/null; then - gen3 kube-setup-indexd & -else - gen3_log_info "no manifest entry for indexd" -fi - if g3k_manifest_lookup .versions.arborist 2> /dev/null; then gen3 kube-setup-arborist || gen3_log_err "arborist setup failed?" else gen3_log_info "no manifest entry for arborist" fi +if g3k_manifest_lookup .versions.indexd 2> /dev/null; then + gen3 kube-setup-indexd & +else + gen3_log_info "no manifest entry for indexd" +fi + if g3k_manifest_lookup '.versions["audit-service"]' 2> /dev/null; then gen3 kube-setup-audit-service else From 229f9a5281819db415d9929c2c2220ec67f3f700 Mon Sep 17 00:00:00 2001 From: George Thomas <98996322+george42-ctds@users.noreply.github.com> Date: Thu, 2 May 2024 09:40:36 -0700 Subject: [PATCH 107/114] HP-1470 Use cedar instance id for mds (#2532) * (HP-1470): use CEDAR instance id for mds queries * (HP-1470): handle case of negative limit from small total --- .../healdata/heal-cedar-data-ingest.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/files/scripts/healdata/heal-cedar-data-ingest.py b/files/scripts/healdata/heal-cedar-data-ingest.py index e0c4b3c463..7b4c638ab4 100644 --- a/files/scripts/healdata/heal-cedar-data-ingest.py +++ b/files/scripts/healdata/heal-cedar-data-ingest.py @@ -227,24 +227,24 @@ def get_related_studies(serial_num, guid, hostname): returned_records = len(metadata_return["metadata"]["records"]) print(f"Successfully got {returned_records} record(s) from CEDAR directory") for cedar_record in metadata_return["metadata"]["records"]: - # get the appl id from cedar for querying in our MDS - cedar_appl_id = pydash.get( - cedar_record, "metadata_location.nih_application_id" + # get the CEDAR instance id from cedar for querying in our MDS + cedar_instance_id = pydash.get( + cedar_record, "metadata_location.cedar_study_level_metadata_template_instance_ID" ) - if cedar_appl_id is None: - print("This record doesn't have appl_id, skipping...") + if cedar_instance_id is None: + print("This record doesn't have CEDAR instance id, skipping...") continue - # Get the metadata record for the nih_application_id + # Get the metadata record for the CEDAR instance id mds = requests.get( - f"http://revproxy-service/mds/metadata?gen3_discovery.study_metadata.metadata_location.nih_application_id={cedar_appl_id}&data=true" + f"http://revproxy-service/mds/metadata?gen3_discovery.study_metadata.metadata_location.cedar_study_level_metadata_template_instance_ID={cedar_instance_id}&data=true" ) if mds.status_code == 200: mds_res = mds.json() # the query result key is the record of the metadata. If it doesn't return anything then our query failed. if len(list(mds_res.keys())) == 0 or len(list(mds_res.keys())) > 1: - print("Query returned nothing for", cedar_appl_id, "appl id") + print(f"Query returned nothing for template_instance_ID={cedar_instance_id}&data=true") continue # get the key for our mds record @@ -394,3 +394,6 @@ def get_related_studies(serial_num, guid, hostname): offset = offset + limit if (offset + limit) > total: limit = total - offset + + if limit < 0: + break From 4cb747f3fd7dfc4c17b593dd42f5ffdb7614d035 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 2 May 2024 14:31:55 -0500 Subject: [PATCH 108/114] MIDRC-639 DICOM viewer v3: fix prefix in config and port (#2533) --- gen3/bin/kube-setup-dicom.sh | 4 ++-- kube/services/dicom-viewer/dicom-viewer-service.yaml | 2 +- kube/services/ohif-viewer/ohif-viewer-deploy.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gen3/bin/kube-setup-dicom.sh b/gen3/bin/kube-setup-dicom.sh index 42110eea2d..e49060ecb6 100644 --- a/gen3/bin/kube-setup-dicom.sh +++ b/gen3/bin/kube-setup-dicom.sh @@ -83,12 +83,12 @@ EOM EOM fi - if g3k_manifest_lookup .versions["dicom-server"] > /dev/null 2>&1; then + if g3k_manifest_lookup '.versions["dicom-server"]' > /dev/null 2>&1; then export DICOM_SERVER_URL="/dicom-server" gen3_log_info "attaching ohif viewer to old dicom-server (orthanc w/ aurora)" fi - if g3k_manifest_lookup .versions["orthanc"] > /dev/null 2>&1; then + if g3k_manifest_lookup .versions.orthanc > /dev/null 2>&1; then export DICOM_SERVER_URL="/orthanc" gen3_log_info "attaching ohif viewer to new dicom-server (orthanc w/ s3)" fi diff --git a/kube/services/dicom-viewer/dicom-viewer-service.yaml b/kube/services/dicom-viewer/dicom-viewer-service.yaml index ea25765845..26f3a21b05 100644 --- a/kube/services/dicom-viewer/dicom-viewer-service.yaml +++ b/kube/services/dicom-viewer/dicom-viewer-service.yaml @@ -12,4 +12,4 @@ spec: nodePort: null name: http type: ClusterIP - \ No newline at end of file + diff --git a/kube/services/ohif-viewer/ohif-viewer-deploy.yaml b/kube/services/ohif-viewer/ohif-viewer-deploy.yaml index fc45434cad..e2df93cd06 100644 --- a/kube/services/ohif-viewer/ohif-viewer-deploy.yaml +++ b/kube/services/ohif-viewer/ohif-viewer-deploy.yaml @@ -86,7 +86,7 @@ spec: periodSeconds: 60 timeoutSeconds: 30 ports: - - containerPort: 80 + - containerPort: 8080 volumeMounts: - name: config-volume-g3auto readOnly: true From 2d3860cfb01d9547c0774a873f38869564a43d1d Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Tue, 7 May 2024 11:04:19 -0400 Subject: [PATCH 109/114] Feat/long running workflow alert (#2538) * Let's see if this works * Fixing typos * Fixing some silliness * Finalizing our workflow monitor * Fixing the branch we point to --- .../workflow-age-monitor/application.yaml | 22 ++++++++ .../argo-workflow-age.yaml | 55 +++++++++++++++++++ kube/services/workflow-age-monitor/auth.yaml | 18 ++++++ 3 files changed, 95 insertions(+) create mode 100644 kube/services/workflow-age-monitor/application.yaml create mode 100644 kube/services/workflow-age-monitor/argo-workflow-age.yaml create mode 100644 kube/services/workflow-age-monitor/auth.yaml diff --git a/kube/services/workflow-age-monitor/application.yaml b/kube/services/workflow-age-monitor/application.yaml new file mode 100644 index 0000000000..99798bb2b8 --- /dev/null +++ b/kube/services/workflow-age-monitor/application.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: argo-workflow-age-monitor-application + namespace: argocd +spec: + destination: + namespace: default + server: https://kubernetes.default.svc + project: default + source: + repoURL: https://github.com/uc-cdis/cloud-automation.git + targetRevision: master + path: kube/services/workflow-age-monitor/ + directory: + exclude: "application.yaml" + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/kube/services/workflow-age-monitor/argo-workflow-age.yaml b/kube/services/workflow-age-monitor/argo-workflow-age.yaml new file mode 100644 index 0000000000..0d0c29115b --- /dev/null +++ b/kube/services/workflow-age-monitor/argo-workflow-age.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: argo-workflow-age + namespace: default +spec: + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + metadata: + labels: + app: gen3job + spec: + serviceAccountName: argo-workflow-monitor + containers: + - name: kubectl + image: quay.io/cdis/awshelper + env: + # This is 3 * 3600, or 3 hours + - name: THRESHOLD_TIME + value: "10800" + - name: SLACK_WEBHOOK_URL + valueFrom: + configMapKeyRef: + name: global + key: slack_webhook + + command: ["/bin/bash"] + args: + - "-c" + - | + #!/bin/bash + # Get all workflows with specific label and check their age + kubectl get workflows --all-namespaces -o json | jq -c '.items[] | {name: .metadata.name, creationTimestamp: .metadata.creationTimestamp}' | while read workflow_info; do + WORKFLOW_NAME=$(echo $workflow_info | jq -r '.name') + CREATION_TIMESTAMP=$(echo $workflow_info | jq -r '.creationTimestamp') + + # Convert creation timestamp to Unix Epoch time + CREATION_EPOCH=$(date -d "$CREATION_TIMESTAMP" +%s) + + # Get current Unix Epoch time + CURRENT_EPOCH=$(date +%s) + + # Calculate workflow age in seconds + WORKFLOW_AGE=$(($CURRENT_EPOCH - $CREATION_EPOCH)) + + # Check if workflow age is greater than threshold + if [ "$WORKFLOW_AGE" -gt "$THRESHOLD_TIME" ]; then + echo "Workflow $WORKFLOW_NAME has been running for over $THRESHOLD_TIME seconds, sending an alert" + # Send alert to Slack + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"WARNING: Workflow \`${WORKFLOW_NAME}\` has been running longer than $THRESHOLD_TIME seconds\"}" $SLACK_WEBHOOK_URL + fi + done + restartPolicy: OnFailure diff --git a/kube/services/workflow-age-monitor/auth.yaml b/kube/services/workflow-age-monitor/auth.yaml new file mode 100644 index 0000000000..fb7970a3ea --- /dev/null +++ b/kube/services/workflow-age-monitor/auth.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: argo-workflow-monitor + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-workflow-monitor-binding +subjects: + - kind: ServiceAccount + name: argo-workflow-monitor + namespace: default +roleRef: + kind: ClusterRole + name: argo-argo-workflows-view + apiGroup: rbac.authorization.k8s.io From 4c467f571a57d43f82895d71cbdec0f5ba61545e Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Thu, 9 May 2024 08:35:53 -0600 Subject: [PATCH 110/114] Update Jenkinsfile to point to GPE-1309 jenkins-lib (#2542) --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 908c2d01a5..0d1f9f34bc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ #!groovy // See 'Loading libraries dynamically' here: https://jenkins.io/doc/book/pipeline/shared-libraries/ -library 'cdis-jenkins-lib@master' +library 'cdis-jenkins-lib@feat/GPE-1309' import org.jenkinsci.plugins.pipeline.modeldefinition.Utils From 01a2b2fe7e0dcdde50bf1a01e1d98594ad46501c Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Fri, 10 May 2024 08:35:01 -0600 Subject: [PATCH 111/114] Update Jenkinsfile (#2545) --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0d1f9f34bc..eaf4dd9c02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ #!groovy // See 'Loading libraries dynamically' here: https://jenkins.io/doc/book/pipeline/shared-libraries/ -library 'cdis-jenkins-lib@feat/GPE-1309' +library 'cdis-jenkins-lib@feat/master' import org.jenkinsci.plugins.pipeline.modeldefinition.Utils From 7ea1380b5a8dab63c939c39cb8005d9800d41a5b Mon Sep 17 00:00:00 2001 From: Aidan Hilt <11202897+AidanHilt@users.noreply.github.com> Date: Mon, 13 May 2024 11:05:41 -0400 Subject: [PATCH 112/114] Adding a purpose label to workflow nodes, so our old node monitoring can catch them (#2546) --- kube/services/argo-events/workflows/configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kube/services/argo-events/workflows/configmap.yaml b/kube/services/argo-events/workflows/configmap.yaml index 8d90457145..cd82478c2c 100644 --- a/kube/services/argo-events/workflows/configmap.yaml +++ b/kube/services/argo-events/workflows/configmap.yaml @@ -75,6 +75,7 @@ data: effect: NoSchedule labels: role: WORKFLOW_NAME + purpose: workflow limits: resources: cpu: 2000 From 6dca75f1dd30a7aca4c2308b45f8cb0829ac473b Mon Sep 17 00:00:00 2001 From: "J. Q" <55899496+jawadqur@users.noreply.github.com> Date: Tue, 14 May 2024 09:28:30 -0500 Subject: [PATCH 113/114] Update web_wildcard_whitelist (#2547) * Update web_wildcard_whitelist * Update Jenkinsfile --- Jenkinsfile | 2 +- files/squid_whitelist/web_wildcard_whitelist | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index eaf4dd9c02..908c2d01a5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ #!groovy // See 'Loading libraries dynamically' here: https://jenkins.io/doc/book/pipeline/shared-libraries/ -library 'cdis-jenkins-lib@feat/master' +library 'cdis-jenkins-lib@master' import org.jenkinsci.plugins.pipeline.modeldefinition.Utils diff --git a/files/squid_whitelist/web_wildcard_whitelist b/files/squid_whitelist/web_wildcard_whitelist index 1374c5d679..1717b44432 100644 --- a/files/squid_whitelist/web_wildcard_whitelist +++ b/files/squid_whitelist/web_wildcard_whitelist @@ -40,6 +40,7 @@ .dockerproject.org .dph.illinois.gov .elasticsearch.org +.eramba.org .erlang-solutions.com .external-secrets.io .extjs.com From ad7d2580f9174e7be8502190d04ccc578a199c24 Mon Sep 17 00:00:00 2001 From: EliseCastle23 <109446148+EliseCastle23@users.noreply.github.com> Date: Tue, 14 May 2024 10:52:47 -0600 Subject: [PATCH 114/114] correcting networkpolicy format for ssjdispatcher and sower (#2539) --- kube/services/netpolicy/gen3/services/sower_netpolicy.yaml | 1 - .../netpolicy/gen3/services/ssjdispatcherjob_netpolicy.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/kube/services/netpolicy/gen3/services/sower_netpolicy.yaml b/kube/services/netpolicy/gen3/services/sower_netpolicy.yaml index 7ad51cacad..93c2de3c33 100644 --- a/kube/services/netpolicy/gen3/services/sower_netpolicy.yaml +++ b/kube/services/netpolicy/gen3/services/sower_netpolicy.yaml @@ -3,7 +3,6 @@ kind: NetworkPolicy metadata: name: netpolicy-sowerjob spec: - spec: podSelector: matchLabels: app: sowerjob diff --git a/kube/services/netpolicy/gen3/services/ssjdispatcherjob_netpolicy.yaml b/kube/services/netpolicy/gen3/services/ssjdispatcherjob_netpolicy.yaml index 7b1f85c291..bd6e03f051 100644 --- a/kube/services/netpolicy/gen3/services/ssjdispatcherjob_netpolicy.yaml +++ b/kube/services/netpolicy/gen3/services/ssjdispatcherjob_netpolicy.yaml @@ -3,7 +3,6 @@ kind: NetworkPolicy metadata: name: netpolicy-ssjdispatcherjob spec: - spec: podSelector: matchLabels: app: ssjdispatcherjob