From bc823321014261eae5b05a73470d558c8c0f92f0 Mon Sep 17 00:00:00 2001 From: bwmac Date: Thu, 20 Jun 2024 12:26:39 -0600 Subject: [PATCH 1/3] updates for 23.4.3 --- .../nextflow-ecs-task-definition.yaml | 25 ++++++++-------- .../nextflow-ecs-task-definition.yaml | 24 +++++++-------- templates/nextflow-ecs-task-definition.j2 | 30 +++++++++++++++++-- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/config/infra-dev/nextflow-ecs-task-definition.yaml b/config/infra-dev/nextflow-ecs-task-definition.yaml index fa8ba96b..6fce1594 100644 --- a/config/infra-dev/nextflow-ecs-task-definition.yaml +++ b/config/infra-dev/nextflow-ecs-task-definition.yaml @@ -6,12 +6,11 @@ dependencies: - infra-dev/nextflow-efs-file-system.yaml - infra-dev/nextflow-elasticache-cluster.yaml - parameters: - TowerSmtpHost: 'email-smtp.us-east-1.amazonaws.com' - TowerSmtpPort: '587' - TowerSmtpUser: !ssm 'smtp-username' - TowerSmtpPassword: !ssm 'smtp-password' + TowerSmtpHost: "email-smtp.us-east-1.amazonaws.com" + TowerSmtpPort: "587" + TowerSmtpUser: !ssm "smtp-username" + TowerSmtpPassword: !ssm "smtp-password" TowerContactEmail: nextflow-admins@sagebase.org TowerServerUrl: https://tower-dev.sagebionetworks.org TowerRedisUrl: !stack_output_external nextflow-elasticache-cluster::RedisEndpoint @@ -23,19 +22,19 @@ parameters: TowerDbPassword: !aws_secrets_manager nextflow-aurora-mysql-NextflowTowerDatabaseUserSecret::SecretString::password TowerGoogleClientId: !aws_secrets_manager nextflow/google_oauth_app::SecretString::client TowerGoogleSecret: !aws_secrets_manager nextflow/google_oauth_app::SecretString::secret - CronContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' - FrontendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' - BackendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + CronContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + FrontendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}" + BackendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + MigrateDBContainerImage: "cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}" EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId - EfsVolumeMountPath: '/efs' - TowerUserWorkspace: 'false' + EfsVolumeMountPath: "/efs" + TowerUserWorkspace: "false" TowerRootUsers: - thomas.yu@sagebase.org - khai.do@sagebase.org - TowerConfigFileName: 'tower.yaml' + TowerConfigFileName: "tower.yaml" -stack_tags: - {{stack_group_config.default_stack_tags}} +stack_tags: { { stack_group_config.default_stack_tags } } sceptre_user_data: environment: !file src/tower/resources/environment.yaml diff --git a/config/infra-prod/nextflow-ecs-task-definition.yaml b/config/infra-prod/nextflow-ecs-task-definition.yaml index ba4df170..76d19e56 100644 --- a/config/infra-prod/nextflow-ecs-task-definition.yaml +++ b/config/infra-prod/nextflow-ecs-task-definition.yaml @@ -7,10 +7,10 @@ dependencies: - infra-prod/nextflow-elasticache-cluster.yaml parameters: - TowerSmtpHost: 'email-smtp.us-east-1.amazonaws.com' - TowerSmtpPort: '587' - TowerSmtpUser: !ssm 'smtp-username' - TowerSmtpPassword: !ssm 'smtp-password' + TowerSmtpHost: "email-smtp.us-east-1.amazonaws.com" + TowerSmtpPort: "587" + TowerSmtpUser: !ssm "smtp-username" + TowerSmtpPassword: !ssm "smtp-password" TowerContactEmail: nextflow-admins@sagebase.org TowerServerUrl: https://tower.sagebionetworks.org TowerRedisUrl: !stack_output_external nextflow-elasticache-cluster::RedisEndpoint @@ -22,18 +22,18 @@ parameters: TowerDbPassword: !aws_secrets_manager nextflow-aurora-mysql-NextflowTowerDatabaseUserSecret::SecretString::password TowerGoogleClientId: !aws_secrets_manager nextflow/google_oauth_app::SecretString::client TowerGoogleSecret: !aws_secrets_manager nextflow/google_oauth_app::SecretString::secret - CronContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' - FrontendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}' - BackendContainerImage: '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}' + CronContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + FrontendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/frontend:{{stack_group_config.tower_version}}" + BackendContainerImage: "cr.seqera.io/private/nf-tower-enterprise/backend:{{stack_group_config.tower_version}}" + MigrateDBContainerImage: "cr.seqera.io/private/nf-tower-enterprise/migrate-db:{{stack_group_config.tower_version}}" EfsFileSystemId: !stack_output_external nextflow-efs-file-system::FileSystemId - EfsVolumeMountPath: '/efs' - TowerUserWorkspace: 'false' + EfsVolumeMountPath: "/efs" + TowerUserWorkspace: "false" TowerRootUsers: - thomas.yu@sagebase.org - TowerConfigFileName: 'tower.yaml' + TowerConfigFileName: "tower.yaml" -stack_tags: - {{stack_group_config.default_stack_tags}} +stack_tags: { { stack_group_config.default_stack_tags } } sceptre_user_data: environment: !file src/tower/resources/environment.yaml diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 0cd9c068..01667b31 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -83,6 +83,15 @@ Parameters: Type: String Description: Redis container docker image, e.g. 'redis:5.0.8' {%- endif %} + MigrateDBContainerName: + Type: String + Description: (Optional) Name of the migrate-db container + Default: migrate-db + MigrateDBContainerImage: + Type: String + Description: > + (Optional) migrate-db container docker image, + e.g. 'cr.seqera.io/private/nf-tower-enterprise/migrate-db:v23.4.3' CronContainerName: Type: String Description: (Optional) Name of the cron container @@ -180,6 +189,21 @@ Resources: EFSVolumeConfiguration: FilesystemId: !Ref EfsFileSystemId ContainerDefinitions: + - image: !Ref RedisContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref MigrateDBContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref FrontendContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref BackendContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + - image: !Ref CronContainerImage + repositoryCredentials: + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' {%- if sceptre_user_data.EnableRedisDocker is defined and sceptre_user_data.EnableRedisDocker %} # The following container definition is a stop-gap solution for # https://sagebionetworks.jira.com/browse/WORKFLOWS-521 @@ -229,8 +253,8 @@ Resources: awslogs-group: !Ref TowerTaskLogGroup awslogs-stream-prefix: !Ref AwslogsStreamPrefix {%- endif %} - - Name: !Sub '${CronContainerName}-MigrateDb' - Image: !Ref CronContainerImage + - Name: !Sub '${MigrateDBContainerName}' + Image: !Ref MigrateDBContainerImage Memory: 2000 Cpu: 0 Essential: false @@ -275,7 +299,7 @@ Resources: - ContainerName: !Ref RedisContainerName Condition: START {%- endif %} - - ContainerName: !Sub '${CronContainerName}-MigrateDb' + - ContainerName: !Sub '${MigrateDBContainerName}' Condition: SUCCESS WorkingDirectory: /work EntryPoint: From df40beb4cdeb91879cb7c12136e13fad8ce71c97 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:04:56 -0600 Subject: [PATCH 2/3] Update nextflow-ecs-task-definition.j2 --- templates/nextflow-ecs-task-definition.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 01667b31..6b233c3a 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -100,7 +100,7 @@ Parameters: Type: String Description: > (Optional) Cron container docker image, - e.g. '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:v21.06.0' + e.g. 'cr.seqera.io/private/nf-tower-enterprise/backend:v21.06.0' FrontendContainerName: Type: String Description: (Optional) Name of the container that runs the tower ui @@ -109,7 +109,7 @@ Parameters: Type: String Description: > Frontend container docker image, - e.g. '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/frontend:v21.06.0' + e.g. 'cr.seqera.io/private/nf-tower-enterprise/frontend:v21.06.0' FrontendContainerPort: Type: Number Description: (Optional) Port to open in frontend container @@ -126,7 +126,7 @@ Parameters: Type: String Description: > Backend container docker image, - e.g. '195996028523.dkr.ecr.eu-west-1.amazonaws.com/nf-tower-enterprise/backend:v21.06.0' + e.g. 'cr.seqera.io/private/nf-tower-enterprise/backend:v21.06.0' BackendContainerPort: Type: Number Description: (Optional) Port to open in backend container @@ -194,10 +194,10 @@ Resources: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - image: !Ref MigrateDBContainerImage repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - image: !Ref FrontendContainerImage repositoryCredentials: - credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' + credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' - image: !Ref BackendContainerImage repositoryCredentials: credentialsParameter: !Sub 'arn:aws:secretsmanager:us-east-1:${AWS::AccountId}:secret:TOWER_DEV_SEQERA_REGISTRY_SECRET' From 05d9a672df8343c8b2e75d3848f07dcb38d207e6 Mon Sep 17 00:00:00 2001 From: Brad Macdonald <52762200+BWMac@users.noreply.github.com> Date: Thu, 20 Jun 2024 14:17:46 -0600 Subject: [PATCH 3/3] Update nextflow-ecs-task-definition.j2 --- templates/nextflow-ecs-task-definition.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/nextflow-ecs-task-definition.j2 b/templates/nextflow-ecs-task-definition.j2 index 6b233c3a..8b82b998 100644 --- a/templates/nextflow-ecs-task-definition.j2 +++ b/templates/nextflow-ecs-task-definition.j2 @@ -253,7 +253,7 @@ Resources: awslogs-group: !Ref TowerTaskLogGroup awslogs-stream-prefix: !Ref AwslogsStreamPrefix {%- endif %} - - Name: !Sub '${MigrateDBContainerName}' + - Name: !Ref MigrateDBContainerName Image: !Ref MigrateDBContainerImage Memory: 2000 Cpu: 0 @@ -299,7 +299,7 @@ Resources: - ContainerName: !Ref RedisContainerName Condition: START {%- endif %} - - ContainerName: !Sub '${MigrateDBContainerName}' + - ContainerName: !Ref MigrateDBContainerName Condition: SUCCESS WorkingDirectory: /work EntryPoint: