Skip to content

Commit

Permalink
feat(stepfunctions-tasks): start glue crawler (#29016)
Browse files Browse the repository at this point in the history
### Issue

Closes #24188.

### Reason for this change

AWS Step Functions supports the ability to start a Glue Crawler as a
task within a state machine. However, this is not configurable.

### Description of changes

I've implemented `GlueStartCrawlerRun` class in stepfunctions-tasks and
we can create start crawler task easily:
```ts
  const task = new GlueStartCrawlerRun(stack, 'Task', {
    crawlerName: 'glue-crawler-name',
  });
```

### Description of how you validated changes

I've added both unit and integ tests.

### Checklist
- [x] My code adheres to the [CONTRIBUTING
GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and
[DESIGN
GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md)

----

*By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license*

---------

Co-authored-by: Luca Pizzini <lpizzini7@gmail.com>
Co-authored-by: GZ <yuanhaoz@amazon.com>
  • Loading branch information
3 people authored Mar 11, 2024
1 parent 840ec97 commit 5592553
Show file tree
Hide file tree
Showing 13 changed files with 1,089 additions and 0 deletions.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
{
"Resources": {
"Bucket83908E77": {
"Type": "AWS::S3::Bucket",
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
},
"GluecrawlwerRole4E24839F": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "glue.amazonaws.com"
}
}
],
"Version": "2012-10-17"
},
"ManagedPolicyArns": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/service-role/AWSGlueServiceRole"
]
]
}
]
}
},
"Database": {
"Type": "AWS::Glue::Database",
"Properties": {
"CatalogId": {
"Ref": "AWS::AccountId"
},
"DatabaseInput": {
"Description": "My database",
"Name": "my-database"
}
}
},
"Crawler": {
"Type": "AWS::Glue::Crawler",
"Properties": {
"DatabaseName": {
"Ref": "Database"
},
"Role": {
"Fn::GetAtt": [
"GluecrawlwerRole4E24839F",
"Arn"
]
},
"Targets": {
"S3Targets": [
{
"Path": {
"Fn::Join": [
"",
[
"s3://",
{
"Ref": "Bucket83908E77"
},
"/"
]
]
}
}
]
}
}
},
"StateMachineRole543B9670": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "states.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
},
"StateMachineRoleDefaultPolicyDA5F7DA8": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": [
"glue:GetCrawler",
"glue:StartCrawler"
],
"Effect": "Allow",
"Resource": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":glue:",
{
"Ref": "AWS::Region"
},
":",
{
"Ref": "AWS::AccountId"
},
":crawler/",
{
"Ref": "Crawler"
}
]
]
}
}
],
"Version": "2012-10-17"
},
"PolicyName": "StateMachineRoleDefaultPolicyDA5F7DA8",
"Roles": [
{
"Ref": "StateMachineRole543B9670"
}
]
}
},
"StateMachine81935E76": {
"Type": "AWS::StepFunctions::StateMachine",
"Properties": {
"DefinitionString": {
"Fn::Join": [
"",
[
"{\"StartAt\":\"Start Task\",\"States\":{\"Start Task\":{\"Type\":\"Pass\",\"Next\":\"Glue Crawler Task\"},\"Glue Crawler Task\":{\"Next\":\"End Task\",\"Type\":\"Task\",\"Resource\":\"arn:",
{
"Ref": "AWS::Partition"
},
":states:::aws-sdk:glue:startCrawler\",\"Parameters\":{\"Name\":\"",
{
"Ref": "Crawler"
},
"\"}},\"End Task\":{\"Type\":\"Pass\",\"End\":true}}}"
]
]
},
"RoleArn": {
"Fn::GetAtt": [
"StateMachineRole543B9670",
"Arn"
]
}
},
"DependsOn": [
"StateMachineRoleDefaultPolicyDA5F7DA8",
"StateMachineRole543B9670"
],
"UpdateReplacePolicy": "Delete",
"DeletionPolicy": "Delete"
}
},
"Parameters": {
"BootstrapVersion": {
"Type": "AWS::SSM::Parameter::Value<String>",
"Default": "/cdk-bootstrap/hnb659fds/version",
"Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
}
},
"Rules": {
"CheckBootstrapVersion": {
"Assertions": [
{
"Assert": {
"Fn::Not": [
{
"Fn::Contains": [
[
"1",
"2",
"3",
"4",
"5"
],
{
"Ref": "BootstrapVersion"
}
]
}
]
},
"AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
}
]
}
}
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5592553

Please sign in to comment.