From e7f6905da153428ac8a2bdcdd766d614a879627a Mon Sep 17 00:00:00 2001 From: Calvin Combs Date: Mon, 15 Apr 2024 16:43:15 -0700 Subject: [PATCH 1/3] unit tests...still need integ --- .../aws-batch/lib/multinode-job-definition.ts | 47 +++++- .../test/multinode-job-definition.test.ts | 143 ++++++++++++++++++ 2 files changed, 187 insertions(+), 3 deletions(-) diff --git a/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts b/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts index a103c8819f1fd..49261c037390d 100644 --- a/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts +++ b/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts @@ -70,8 +70,11 @@ export interface MultiNodeContainer { * The index of the last node to run this container. * * The container is run on all nodes in the range [startNode, endNode] (inclusive) + * + * If this value is not specified, you must specify `numNodes`. + * @default - Batch assigns the highest possible node index. */ - readonly endNode: number; + readonly endNode?: number; /** * The container that this node range will run @@ -108,6 +111,14 @@ export interface MultiNodeJobDefinitionProps extends JobDefinitionProps { */ readonly mainNode?: number; + /** + * The total number of nodes used in this job. + * **Only specify if there is at least one container for which you have not specified `endNode`.** + * + * @default - computed based on the containers passed in. + */ + readonly numNodes?: number; + /** * Whether to propogate tags from the JobDefinition * to the ECS task that Batch spawns @@ -165,14 +176,14 @@ export class MultiNodeJobDefinition extends JobDefinitionBase implements IMultiN mainNode: this.mainNode ?? 0, nodeRangeProperties: Lazy.any({ produce: () => this.containers.map((container) => ({ - targetNodes: container.startNode + ':' + container.endNode, + targetNodes: container.startNode + ':' + (container.endNode ?? ''), container: { ...container.container._renderContainerDefinition(), instanceType: this._instanceType?.toString(), }, })), }), - numNodes: Lazy.number({ + numNodes: props?.numNodes ?? Lazy.number({ produce: () => computeNumNodes(this.containers), }), }, @@ -186,6 +197,7 @@ export class MultiNodeJobDefinition extends JobDefinitionBase implements IMultiN this.jobDefinitionName = this.getResourceNameAttribute(resource.ref); this.node.addValidation({ validate: () => validateContainers(this.containers) }); + this.node.addValidation({ validate: () => validateNumNodesPropConflicts(this.containers, this.node.id, props?.numNodes) }); } /** @@ -209,6 +221,9 @@ function computeNumNodes(containers: MultiNodeContainer[]) { let result = 0; for (const container of containers) { + if (!container.endNode) { + throw new Error(`The multinode container '${container.container.node.id}' does not specify an end node, and its Job Definition does not specify 'numNodes'. Please either specify 'numNodes' or specify 'endNode' for every container in this Job Definition.`); + } result += container.endNode - container.startNode + 1; } @@ -218,3 +233,29 @@ function computeNumNodes(containers: MultiNodeContainer[]) { function validateContainers(containers: MultiNodeContainer[]): string[] { return containers.length === 0 ? ['multinode job has no containers!'] : []; } + +function validateNumNodesPropConflicts(containers: MultiNodeContainer[], jobDefnName: string, numNodes?: number): string[] { + let allContainersSpecifyEndNode = true; + let noEndNodeContainers = []; + for (const container of containers ?? []) { + if (!container.endNode) { + allContainersSpecifyEndNode = false; + noEndNodeContainers.push(container.container.node.id); + } + } + + if (numNodes && allContainersSpecifyEndNode) { + return [`All containers of Multinode Job Definition '${jobDefnName}' specify 'endNode', but the job definition specifies 'numNodes'! Do not specify 'endNode' for every container with 'numNodes', the CDK will compute the correct value for 'numNodes' if all containers have 'endNode' specified.`]; + } + + const returnValue = []; + + if (!numNodes && !allContainersSpecifyEndNode) { + for (const containerId of noEndNodeContainers) { + returnValue.push(`The multinode container '${containerId}' does not specify an end node, and its Job Definition does not + specify 'numNodes'. Please either specify 'numNodes' or specify 'endNode' for every container in this Job Definition.`); + } + } + + return []; +} diff --git a/packages/aws-cdk-lib/aws-batch/test/multinode-job-definition.test.ts b/packages/aws-cdk-lib/aws-batch/test/multinode-job-definition.test.ts index 0e5fb4cd3395c..8833868245c32 100644 --- a/packages/aws-cdk-lib/aws-batch/test/multinode-job-definition.test.ts +++ b/packages/aws-cdk-lib/aws-batch/test/multinode-job-definition.test.ts @@ -93,6 +93,87 @@ test('MultiNodeJobDefinition respects instanceType', () => { }); }); +test('MultiNodeJobDefinition respects numNodes', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + new MultiNodeJobDefinition(stack, 'ECSJobDefn', { + containers: [{ + container: new EcsEc2ContainerDefinition(stack, 'MultinodeContainer', { + cpu: 256, + memory: Size.mebibytes(2048), + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + }), + startNode: 0, + }], + numNodes: 10, + instanceType: InstanceType.of(InstanceClass.R4, InstanceSize.LARGE), + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Batch::JobDefinition', { + NodeProperties: { + NodeRangeProperties: [{ + Container: { + }, + TargetNodes: '0:', + }], + NumNodes: 10, + }, + PlatformCapabilities: [Compatibility.EC2], + }); +}); + +test('MultiNodeJobDefinition respects numNodes with two containers', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + const jobDefn = new MultiNodeJobDefinition(stack, 'ECSJobDefn', { + containers: [{ + container: new EcsEc2ContainerDefinition(stack, 'MultinodeContainer', { + cpu: 256, + memory: Size.mebibytes(2048), + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + }), + startNode: 0, + endNode: 5, + }], + numNodes: 10, + instanceType: InstanceType.of(InstanceClass.R4, InstanceSize.LARGE), + }); + + jobDefn.addContainer({ + container: new EcsEc2ContainerDefinition(stack, 'MultinodeContainer2', { + cpu: 256, + memory: Size.mebibytes(2048), + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + }), + startNode: 6, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Batch::JobDefinition', { + NodeProperties: { + NodeRangeProperties: [ + { + Container: { + }, + TargetNodes: '0:5', + }, + { + Container: { + }, + TargetNodes: '6:', + }, + ], + NumNodes: 10, + }, + PlatformCapabilities: [Compatibility.EC2], + }); +}); + test('MultiNodeJobDefinition one container', () => { // GIVEN const stack = new Stack(); @@ -191,6 +272,68 @@ test('multinode job requires at least one container', () => { expect(() => Template.fromStack(stack)).toThrow(/multinode job has no containers!/); }); +test('multinode job does not allow specifying all containers with `endNode` if `numNodes` is specified', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + const jobDefn = new MultiNodeJobDefinition(stack, 'ECSJobDefn', { + numNodes: 10, + containers: [{ + container: new EcsEc2ContainerDefinition(stack, 'MultinodeContainer1', { + cpu: 256, + memory: Size.mebibytes(2048), + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + }), + startNode: 0, + endNode: 5, + }], + }); + jobDefn.addContainer({ + container: new EcsEc2ContainerDefinition(stack, 'MultinodeContainer2', { + cpu: 256, + memory: Size.mebibytes(2048), + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + }), + startNode: 6, + endNode: 10, + }); + + // THEN + expect(() => Template.fromStack(stack)).toThrow(/All containers of Multinode Job Definition 'ECSJobDefn' specify 'endNode', but the job definition specifies 'numNodes'!/); +}); + +test('MultiNodeJobDefinition throws some dumb error somewhere', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + const jobDefn = new MultiNodeJobDefinition(stack, 'ECSJobDefn', { + containers: [{ + container: new EcsEc2ContainerDefinition(stack, 'MultinodeContainer', { + cpu: 256, + memory: Size.mebibytes(2048), + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + }), + startNode: 0, + }], + instanceType: InstanceType.of(InstanceClass.R4, InstanceSize.LARGE), + }); + + jobDefn.addContainer({ + container: new EcsEc2ContainerDefinition(stack, 'MultinodeContainer2', { + cpu: 256, + memory: Size.mebibytes(2048), + image: ecs.ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + }), + startNode: 6, + endNode: 10, + }); + + // THEN + expect(() => Template.fromStack(stack)).toThrow(/The multinode container 'MultinodeContainer' does not specify an end node, and its Job Definition does not specify 'numNodes'/); +}); + test('multinode job returns a dummy instance type when accessing `instanceType`', () => { // GIVEN const stack = new Stack(); From 12b87f6d18434859c820570e36d97ba5c69418a5 Mon Sep 17 00:00:00 2001 From: Calvin Combs Date: Tue, 16 Apr 2024 09:04:08 -0700 Subject: [PATCH 2/3] snaps --- ...efaultTestDeployAssertDB834B9D.assets.json | 2 +- .../cdk.out | 2 +- .../integ.json | 2 +- .../manifest.json | 48 +- .../stack.assets.json | 8 +- .../stack.template.json | 270 +++++++++ .../tree.json | 543 ++++++++++++++++-- .../test/integ.multinode-job-definition.ts | 31 + 8 files changed, 861 insertions(+), 45 deletions(-) diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/BatchMultiNodeJobDefinitionTestDefaultTestDeployAssertDB834B9D.assets.json b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/BatchMultiNodeJobDefinitionTestDefaultTestDeployAssertDB834B9D.assets.json index 611d2e475dd6e..4ee053a2157a2 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/BatchMultiNodeJobDefinitionTestDefaultTestDeployAssertDB834B9D.assets.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/BatchMultiNodeJobDefinitionTestDefaultTestDeployAssertDB834B9D.assets.json @@ -1,5 +1,5 @@ { - "version": "34.0.0", + "version": "36.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/cdk.out b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/cdk.out index 2313ab5436501..1f0068d32659a 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/cdk.out +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"34.0.0"} \ No newline at end of file +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/integ.json b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/integ.json index 6441e01b3daff..0b1cdc4a0ee31 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/integ.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "34.0.0", + "version": "36.0.0", "testCases": { "BatchMultiNodeJobDefinitionTest/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/manifest.json b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/manifest.json index 9f844bd585d06..f26d3aa2b03df 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/manifest.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "34.0.0", + "version": "36.0.0", "artifacts": { "stack.assets": { "type": "cdk:asset-manifest", @@ -14,10 +14,11 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "stack.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e0a079fad9490e60650bca4a35c6c90273204a816beb74fb80c7b5ae490208d1.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/536df3942835725f00ffdc81c12ff4c49236d191fc8eda8a1e07a000b3d88779.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -81,6 +82,48 @@ "data": "multiContainerExecutionRoleDefaultPolicyE3E7D32C" } ], + "/stack/unlimitednodecontainer1/ExecutionRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "unlimitednodecontainer1ExecutionRole296EB7B9" + } + ], + "/stack/unlimitednodecontainer1/ExecutionRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "unlimitednodecontainer1ExecutionRoleDefaultPolicy6D5ED087" + } + ], + "/stack/UnlimitedNodeContainerJob/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "UnlimitedNodeContainerJob74FE19A6" + } + ], + "/stack/unlimitednodecontainer2/ExecutionRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "unlimitednodecontainer2ExecutionRoleD2524EBD" + } + ], + "/stack/unlimitednodecontainer2/ExecutionRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "unlimitednodecontainer2ExecutionRoleDefaultPolicyFB6CAE06" + } + ], + "/stack/unlimmitedcontainer3/ExecutionRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "unlimmitedcontainer3ExecutionRoleB159462D" + } + ], + "/stack/unlimmitedcontainer3/ExecutionRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "unlimmitedcontainer3ExecutionRoleDefaultPolicy280AA69A" + } + ], "/stack/BootstrapVersion": [ { "type": "aws:cdk:logicalId", @@ -109,6 +152,7 @@ "environment": "aws://unknown-account/unknown-region", "properties": { "templateFile": "BatchMultiNodeJobDefinitionTestDefaultTestDeployAssertDB834B9D.template.json", + "terminationProtection": false, "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.assets.json b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.assets.json index 833c663116fea..f457690941ddf 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.assets.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.assets.json @@ -1,8 +1,7 @@ { - "version": "34.0.0", + "version": "36.0.0", "files": { - "e0a079fad9490e60650bca4a35c6c90273204a816beb74fb80c7b5ae490208d1": { - + "536df3942835725f00ffdc81c12ff4c49236d191fc8eda8a1e07a000b3d88779": { "source": { "path": "stack.template.json", "packaging": "file" @@ -10,8 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "e0a079fad9490e60650bca4a35c6c90273204a816beb74fb80c7b5ae490208d1.json", - + "objectKey": "536df3942835725f00ffdc81c12ff4c49236d191fc8eda8a1e07a000b3d88779.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.template.json b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.template.json index 212323df6324b..85db259c16280 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.template.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/stack.template.json @@ -288,6 +288,276 @@ } ] } + }, + "unlimitednodecontainer1ExecutionRole296EB7B9": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "unlimitednodecontainer1ExecutionRoleDefaultPolicy6D5ED087": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/batch/job:*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "unlimitednodecontainer1ExecutionRoleDefaultPolicy6D5ED087", + "Roles": [ + { + "Ref": "unlimitednodecontainer1ExecutionRole296EB7B9" + } + ] + } + }, + "UnlimitedNodeContainerJob74FE19A6": { + "Type": "AWS::Batch::JobDefinition", + "Properties": { + "NodeProperties": { + "MainNode": 0, + "NodeRangeProperties": [ + { + "Container": { + "Environment": [], + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "unlimitednodecontainer1ExecutionRole296EB7B9", + "Arn" + ] + }, + "Image": "amazon/amazon-ecs-sample", + "ReadonlyRootFilesystem": false, + "ResourceRequirements": [ + { + "Type": "MEMORY", + "Value": "2048" + }, + { + "Type": "VCPU", + "Value": "256" + } + ] + }, + "TargetNodes": "0:" + }, + { + "Container": { + "Environment": [], + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "unlimitednodecontainer2ExecutionRoleD2524EBD", + "Arn" + ] + }, + "Image": "amazon/amazon-ecs-sample", + "ReadonlyRootFilesystem": false, + "ResourceRequirements": [ + { + "Type": "MEMORY", + "Value": "2048" + }, + { + "Type": "VCPU", + "Value": "256" + } + ] + }, + "TargetNodes": "11:" + }, + { + "Container": { + "Environment": [], + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "unlimmitedcontainer3ExecutionRoleB159462D", + "Arn" + ] + }, + "Image": "amazon/amazon-ecs-sample", + "ReadonlyRootFilesystem": false, + "ResourceRequirements": [ + { + "Type": "MEMORY", + "Value": "2048" + }, + { + "Type": "VCPU", + "Value": "256" + } + ] + }, + "TargetNodes": "15:" + } + ], + "NumNodes": 20 + }, + "PlatformCapabilities": [ + "EC2" + ], + "PropagateTags": true, + "RetryStrategy": {}, + "Timeout": {}, + "Type": "multinode" + } + }, + "unlimitednodecontainer2ExecutionRoleD2524EBD": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "unlimitednodecontainer2ExecutionRoleDefaultPolicyFB6CAE06": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/batch/job:*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "unlimitednodecontainer2ExecutionRoleDefaultPolicyFB6CAE06", + "Roles": [ + { + "Ref": "unlimitednodecontainer2ExecutionRoleD2524EBD" + } + ] + } + }, + "unlimmitedcontainer3ExecutionRoleB159462D": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "unlimmitedcontainer3ExecutionRoleDefaultPolicy280AA69A": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/batch/job:*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "unlimmitedcontainer3ExecutionRoleDefaultPolicy280AA69A", + "Roles": [ + { + "Ref": "unlimmitedcontainer3ExecutionRoleB159462D" + } + ] + } } }, "Parameters": { diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/tree.json b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/tree.json index 27886a65ba0ae..3e5537eef1d02 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/tree.json +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.js.snapshot/tree.json @@ -21,8 +21,7 @@ "path": "stack/myContainer/ExecutionRole/ImportExecutionRole", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" - + "version": "10.3.0" } }, "Resource": { @@ -47,7 +46,7 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "DefaultPolicy": { @@ -102,19 +101,19 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "batchDefaultLogGroup": { @@ -122,13 +121,13 @@ "path": "stack/myContainer/batchDefaultLogGroup", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "SingleContainerMultiNodeJob": { @@ -182,13 +181,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "multinodecontainer": { @@ -204,7 +203,7 @@ "path": "stack/multinodecontainer/ExecutionRole/ImportExecutionRole", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "Resource": { @@ -229,7 +228,7 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "DefaultPolicy": { @@ -284,19 +283,19 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "batchDefaultLogGroup": { @@ -304,13 +303,13 @@ "path": "stack/multinodecontainer/batchDefaultLogGroup", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "MultiContainerMultiNodeJob": { @@ -389,13 +388,13 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "multiContainer": { @@ -411,7 +410,7 @@ "path": "stack/multiContainer/ExecutionRole/ImportExecutionRole", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "Resource": { @@ -436,7 +435,7 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "DefaultPolicy": { @@ -491,19 +490,19 @@ }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "batchDefaultLogGroup": { @@ -511,13 +510,487 @@ "path": "stack/multiContainer/batchDefaultLogGroup", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" + } + }, + "unlimitednodecontainer1": { + "id": "unlimitednodecontainer1", + "path": "stack/unlimitednodecontainer1", + "children": { + "ExecutionRole": { + "id": "ExecutionRole", + "path": "stack/unlimitednodecontainer1/ExecutionRole", + "children": { + "ImportExecutionRole": { + "id": "ImportExecutionRole", + "path": "stack/unlimitednodecontainer1/ExecutionRole/ImportExecutionRole", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "Resource": { + "id": "Resource", + "path": "stack/unlimitednodecontainer1/ExecutionRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "stack/unlimitednodecontainer1/ExecutionRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "stack/unlimitednodecontainer1/ExecutionRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/batch/job:*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "policyName": "unlimitednodecontainer1ExecutionRoleDefaultPolicy6D5ED087", + "roles": [ + { + "Ref": "unlimitednodecontainer1ExecutionRole296EB7B9" + } + ] + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "batchDefaultLogGroup": { + "id": "batchDefaultLogGroup", + "path": "stack/unlimitednodecontainer1/batchDefaultLogGroup", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "UnlimitedNodeContainerJob": { + "id": "UnlimitedNodeContainerJob", + "path": "stack/UnlimitedNodeContainerJob", + "children": { + "Resource": { + "id": "Resource", + "path": "stack/UnlimitedNodeContainerJob/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Batch::JobDefinition", + "aws:cdk:cloudformation:props": { + "nodeProperties": { + "mainNode": 0, + "nodeRangeProperties": [ + { + "targetNodes": "0:", + "container": { + "image": "amazon/amazon-ecs-sample", + "environment": [], + "executionRoleArn": { + "Fn::GetAtt": [ + "unlimitednodecontainer1ExecutionRole296EB7B9", + "Arn" + ] + }, + "readonlyRootFilesystem": false, + "resourceRequirements": [ + { + "type": "MEMORY", + "value": "2048" + }, + { + "type": "VCPU", + "value": "256" + } + ] + } + }, + { + "targetNodes": "11:", + "container": { + "image": "amazon/amazon-ecs-sample", + "environment": [], + "executionRoleArn": { + "Fn::GetAtt": [ + "unlimitednodecontainer2ExecutionRoleD2524EBD", + "Arn" + ] + }, + "readonlyRootFilesystem": false, + "resourceRequirements": [ + { + "type": "MEMORY", + "value": "2048" + }, + { + "type": "VCPU", + "value": "256" + } + ] + } + }, + { + "targetNodes": "15:", + "container": { + "image": "amazon/amazon-ecs-sample", + "environment": [], + "executionRoleArn": { + "Fn::GetAtt": [ + "unlimmitedcontainer3ExecutionRoleB159462D", + "Arn" + ] + }, + "readonlyRootFilesystem": false, + "resourceRequirements": [ + { + "type": "MEMORY", + "value": "2048" + }, + { + "type": "VCPU", + "value": "256" + } + ] + } + } + ], + "numNodes": 20 + }, + "platformCapabilities": [ + "EC2" + ], + "propagateTags": true, + "retryStrategy": {}, + "timeout": {}, + "type": "multinode" + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "unlimitednodecontainer2": { + "id": "unlimitednodecontainer2", + "path": "stack/unlimitednodecontainer2", + "children": { + "ExecutionRole": { + "id": "ExecutionRole", + "path": "stack/unlimitednodecontainer2/ExecutionRole", + "children": { + "ImportExecutionRole": { + "id": "ImportExecutionRole", + "path": "stack/unlimitednodecontainer2/ExecutionRole/ImportExecutionRole", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "Resource": { + "id": "Resource", + "path": "stack/unlimitednodecontainer2/ExecutionRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "stack/unlimitednodecontainer2/ExecutionRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "stack/unlimitednodecontainer2/ExecutionRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/batch/job:*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "policyName": "unlimitednodecontainer2ExecutionRoleDefaultPolicyFB6CAE06", + "roles": [ + { + "Ref": "unlimitednodecontainer2ExecutionRoleD2524EBD" + } + ] + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "batchDefaultLogGroup": { + "id": "batchDefaultLogGroup", + "path": "stack/unlimitednodecontainer2/batchDefaultLogGroup", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "unlimmitedcontainer3": { + "id": "unlimmitedcontainer3", + "path": "stack/unlimmitedcontainer3", + "children": { + "ExecutionRole": { + "id": "ExecutionRole", + "path": "stack/unlimmitedcontainer3/ExecutionRole", + "children": { + "ImportExecutionRole": { + "id": "ImportExecutionRole", + "path": "stack/unlimmitedcontainer3/ExecutionRole/ImportExecutionRole", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "Resource": { + "id": "Resource", + "path": "stack/unlimmitedcontainer3/ExecutionRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "stack/unlimmitedcontainer3/ExecutionRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "stack/unlimmitedcontainer3/ExecutionRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":logs:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":log-group:/aws/batch/job:*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "policyName": "unlimmitedcontainer3ExecutionRoleDefaultPolicy280AA69A", + "roles": [ + { + "Ref": "unlimmitedcontainer3ExecutionRoleB159462D" + } + ] + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "batchDefaultLogGroup": { + "id": "batchDefaultLogGroup", + "path": "stack/unlimmitedcontainer3/batchDefaultLogGroup", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" } }, "BootstrapVersion": { @@ -525,7 +998,7 @@ "path": "stack/BootstrapVersion", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "CheckBootstrapVersion": { @@ -533,13 +1006,13 @@ "path": "stack/CheckBootstrapVersion", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "BatchMultiNodeJobDefinitionTest": { @@ -555,7 +1028,7 @@ "path": "BatchMultiNodeJobDefinitionTest/DefaultTest/Default", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "DeployAssert": { @@ -567,7 +1040,7 @@ "path": "BatchMultiNodeJobDefinitionTest/DefaultTest/DeployAssert/BootstrapVersion", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } }, "CheckBootstrapVersion": { @@ -575,13 +1048,13 @@ "path": "BatchMultiNodeJobDefinitionTest/DefaultTest/DeployAssert/CheckBootstrapVersion", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, @@ -601,13 +1074,13 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } }, "constructInfo": { "fqn": "constructs.Construct", - "version": "10.2.70" + "version": "10.3.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.ts b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.ts index 461881c870400..d0a326cad90b2 100644 --- a/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.ts +++ b/packages/@aws-cdk-testing/framework-integ/test/aws-batch/test/integ.multinode-job-definition.ts @@ -43,6 +43,37 @@ multinodeJob.addContainer({ }), }); +const unlimitedMultinodeJob = new batch.MultiNodeJobDefinition(stack, 'UnlimitedNodeContainerJob', { + containers: [{ + startNode: 0, + container: new batch.EcsEc2ContainerDefinition(stack, 'unlimitednodecontainer1', { + image: ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + cpu: 256, + memory: Size.mebibytes(2048), + }), + }], + propagateTags: true, + numNodes: 20, +}); + +unlimitedMultinodeJob.addContainer({ + startNode: 11, + container: new batch.EcsEc2ContainerDefinition(stack, 'unlimitednodecontainer2', { + image: ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + cpu: 256, + memory: Size.mebibytes(2048), + }), +}); + +unlimitedMultinodeJob.addContainer({ + startNode: 15, + container: new batch.EcsEc2ContainerDefinition(stack, 'unlimmitedcontainer3', { + image: ContainerImage.fromRegistry('amazon/amazon-ecs-sample'), + cpu: 256, + memory: Size.mebibytes(2048), + }), +}); + new integ.IntegTest(app, 'BatchMultiNodeJobDefinitionTest', { testCases: [stack], }); From 798bc235771de0eab3105dbc5e44bb59882c9c4c Mon Sep 17 00:00:00 2001 From: Calvin Combs Date: Tue, 16 Apr 2024 11:19:19 -0700 Subject: [PATCH 3/3] comment...but this change doesn't really make sense --- packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts b/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts index 49261c037390d..f4c1b01e3a590 100644 --- a/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts +++ b/packages/aws-cdk-lib/aws-batch/lib/multinode-job-definition.ts @@ -115,7 +115,7 @@ export interface MultiNodeJobDefinitionProps extends JobDefinitionProps { * The total number of nodes used in this job. * **Only specify if there is at least one container for which you have not specified `endNode`.** * - * @default - computed based on the containers passed in. + * @default - computed from the `startNode` and `endNode` on all containers added to this job definition. */ readonly numNodes?: number;