Merge branch 'main' into feat/ray-orchestrator-sfn

awslabs · Oct 22, 2024 · dbff0cb · dbff0cb
2 parents 7573e8c + edc66f1
commit dbff0cb
Show file tree

Hide file tree

Showing 13 changed files with 659 additions and 61 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,7 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## UNRELEASED
 
 ### **Added**
+
 - added `ray-orchestrator` module
+- added GitHub as alternate option for code repository support along with AWS CodeCommit for sagemaker-templates-service-catalog module
 
 ### **Changed**
 - updated manifests to idf release 1.12.0

diff --git a/examples/manifests/sagemaker-templates-modules-codecommit.yaml b/examples/manifests/sagemaker-templates-modules-codecommit.yaml
@@ -0,0 +1,78 @@
+name: service-catalog
+path: modules/sagemaker/sagemaker-templates-service-catalog
+targetAccount: primary
+parameters:
+  - name: repository-type
+    value: CodeCommit
+  - name: portfolio-access-role-arn
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: LeadDataScientistRoleArn
+  - name: dev-account-id
+    valueFrom:
+      envVariable: PRIMARY_ACCOUNT
+  - name: dev-region
+    valueFrom:
+      envVariable: PRIMARY_REGION
+  - name: dev-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: dev-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: pre-prod-account-id
+    valueFrom:
+      envVariable: PRIMARY_ACCOUNT
+  - name: pre-prod-region
+    valueFrom:
+      envVariable: PRIMARY_REGION
+  - name: pre-prod-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: pre-prod-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: prod-account-id
+    valueFrom:
+      envVariable: PRIMARY_ACCOUNT
+  - name: prod-region
+    valueFrom:
+      envVariable: PRIMARY_REGION
+  - name: prod-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: prod-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: sagemaker-domain-id
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: StudioDomainId
+  - name: sagemaker-domain-arn
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: StudioDomainArn
diff --git a/examples/manifests/sagemaker-templates-modules-github.yaml b/examples/manifests/sagemaker-templates-modules-github.yaml
@@ -0,0 +1,84 @@
+name: service-catalog
+path: modules/sagemaker/sagemaker-templates-service-catalog
+targetAccount: primary
+parameters:
+  - name: repository-type
+    value: GitHub
+  - name: repository-owner
+    value: <github_account_owner>
+  - name: access-token-secret-name
+    value: github_token
+  - name: aws-codeconnection-arn
+    value: <arn>
+  - name: portfolio-access-role-arn
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: LeadDataScientistRoleArn
+  - name: dev-account-id
+    valueFrom:
+      envVariable: PRIMARY_ACCOUNT
+  - name: dev-region
+    valueFrom:
+      envVariable: PRIMARY_REGION
+  - name: dev-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: dev-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: pre-prod-account-id
+    valueFrom:
+      envVariable: PRIMARY_ACCOUNT
+  - name: pre-prod-region
+    valueFrom:
+      envVariable: PRIMARY_REGION
+  - name: pre-prod-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: pre-prod-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: prod-account-id
+    valueFrom:
+      envVariable: PRIMARY_ACCOUNT
+  - name: prod-region
+    valueFrom:
+      envVariable: PRIMARY_REGION
+  - name: prod-vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: VpcId
+  - name: prod-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: networking
+        key: PrivateSubnetIds
+  - name: sagemaker-domain-id
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: StudioDomainId
+  - name: sagemaker-domain-arn
+    valueFrom:
+      moduleMetadata:
+        group: sagemaker-studio
+        name: studio
+        key: StudioDomainArn
diff --git a/manifests/mlops-sagemaker/sagemaker-templates-modules.yaml b/manifests/mlops-sagemaker/sagemaker-templates-modules.yaml
@@ -1,5 +1,6 @@
 name: service-catalog
-path: git::https://github.com/awslabs/aiops-modules.git//modules/sagemaker/sagemaker-templates-service-catalog?ref=release/1.6.0&depth=1
+
+path: modules/sagemaker/sagemaker-templates-service-catalog
 targetAccount: primary
 parameters:
   - name: portfolio-access-role-arn

diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/README.md b/modules/sagemaker/sagemaker-templates-service-catalog/README.md
@@ -49,11 +49,27 @@ The template contains an example CI/CD pipeline to deploy the model endpoints to
 
 The template is based on basic multi-account template from [AWS Enterprise MLOps Framework](https://github.com/aws-samples/aws-enterprise-mlops-framework/blob/main/mlops-multi-account-cdk/mlops-sm-project-template/README.md#sagemaker-project-stack).
 
+#### Third-party Code Repository Integration 
+SageMaker templates support third party code repository (GitHub) integration along with default AWS CodeCommit. As part of integration, SageMaker templates will be able to manage (create, delete) repositories. As an example, if `sagemaker-templates-service-catalog` template configured to use GitHub as repository type then it would create code repository directly into GitHub account provided with manifest configuration. Repository will be named after SageMaker project name in AWS account `{sagemaker-project}-deploy`. For example, if SageMaker project name is `aiops-abalone-model` then GitHub repository would be created with name `aiops-abalone-model-deploy`.
+
+
+## Prerequesites:
+### AWS CodeCommit repository integration
+- There isn't any prerequesite for using CodeCommit repository with SageMaker templates. It is supported as default repository.
+> [!IMPORTANT] 
+> It is important to note AWS CodeCommit is no longer available to new customers. Existing customers of AWS CodeCommit can continue to use the service as normal. 
+### GitHub repository integration
+- Target AWS account should contain AWS Secret Manager secret that contains GitHub personal access token with required permissions to manage repository. Refer guide [Creating a fine-grained personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) in order to create access token.
+- Template also requires AWS CodeConnection created for GitHub provider in order to integrated GitHub repositories AWS CodeBuild and AWS CodePipeline. Refer guide [Create a connection to GitHub](https://docs.aws.amazon.com/dtconsole/latest/userguide/connections-create-github.html) in order to create connection with GitHub.
+
 ## Inputs and outputs:
 ### Required inputs:
   - `portfolio-access-role-arn` - the ARN of the IAM Role used to access the Service Catalog Portfolio or SageMaker projects
-
 ### Optional Inputs:
+  - `repository-type` - type of repository to be integrated with Sagemaker template source code, exp. `GitHub`. If `CodeCommit` is provided then other GitHub repository params are ignored. This is optional parameter, if not provided `CodeCommit` is set as default
+  - `repository-owner` - owner or organisation of project code repository 
+  - `access-token-secret-name` - AWS Secret Manager secret name where access token is stored, this is used to manage repository from template
+  - `aws-codeconnection-arn` -  AWS CodeConnection ARN for repository provider, currently template supports GitHub provider
   - `portfolio-name` - name of the Service Catalog Portfolio
   - `portfolio-owner` - owner of the Service Catalog Portfolio
   - `dev-vpc-id` - id of VPC in dev environment
@@ -153,6 +169,9 @@ parameters:
         name: studio
         key: StudioDomainArn
 ```
+### Sample manifest example for source repository options
+[sagemaker-templates-modules-github.yaml](/examples/manifests/sagemaker-templates-modules-github.yaml)
+[sagemaker-templates-modules-codecommit.yaml](/examples/manifests/sagemaker-templates-modules-codecommit.yaml)
 
 ### Outputs (module metadata):
   - `ServiceCatalogPortfolioName` - the name of the Service Catalog Portfolio

diff --git a/modules/sagemaker/sagemaker-templates-service-catalog/common/__init__.py b/modules/sagemaker/sagemaker-templates-service-catalog/common/__init__.py