chore: Split Stateful Single Stack to Multi Stack (#216)

.prettierignore

@@ -30,7 +30,6 @@ config/event_schemas/
 skel/
 docs/
 openapi/
-shared/
 venv/
 
 # TODO still early days let ignore prettier them (microservice apps) for now

README.md

@@ -103,12 +103,25 @@ _Heads up: Polyglot programming environment. We shorten some trivial steps into
 
 To develop your microservice application please read the [microservice guide](docs/developer/MICROSERVICE.md).
 
-Do note that we have some shared resources that is expected to be used across microservices at [shared resource docs](docs/developer/SHARED_RESOURCES.md).
+Do note that we have some shared resources that is expected to be used across microservices at [shared resource docs](./lib/workload/stateful/stacks/shared/README.md).
 
 ### Typography
 
 When possible, please use either `OrcaBus` (camel case) or `orcabus` (all lower case).
 
+#### Typescript
+
+When using typescript we will use the convention defined in [AWS
+Guide](https://docs.aws.amazon.com/prescriptive-guidance/latest/best-practices-cdk-typescript-iac/typescript-best-practices.html#naming-conventions).
+
+- Use camelCase for variable and function names.
+- Use PascalCase for class names and interface names.
+- Use camelCase for interface members.
+- Use PascalCase for type names and enum names.
+- Name files with camelCase (for example, ebsVolumes.tsx or storage.tsb)
+
+For folder name, we will be using `kebab-case` as this is the common convention in TypeScript project.
+
 ### Toolchain
 
 _Setting up baseline toolchain_

bin/stateful-pipeline.ts → bin/statefulPipeline.ts

@@ -2,7 +2,7 @@
 import 'source-map-support/register';
 
 import * as cdk from 'aws-cdk-lib';
-import { StatefulPipelineStack } from '../lib/pipeline/orcabus-stateful-pipeline-stack';
+import { StatefulPipelineStack } from '../lib/pipeline/statefulPipelineStack';
 
 const AWS_TOOLCHAIN_ACCOUNT = '383856791668'; // Bastion
 const AWS_TOOLCHAIN_REGION = 'ap-southeast-2';

config/constants.ts

@@ -1,19 +1,22 @@
-import { OrcaBusStatefulConfig } from '../lib/workload/orcabus-stateful-stack';
 import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds';
 import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack';
 import { Duration, RemovalPolicy } from 'aws-cdk-lib';
-import { EventSourceProps } from '../lib/workload/stateful/event_source/component';
+import { EventSourceProps } from '../lib/workload/stateful/stacks/shared/constructs/event-source';
 import { DbAuthType } from '../lib/workload/stateless/postgres_manager/function/type';
 import {
   FILEMANAGER_SERVICE_NAME,
   FilemanagerConfig,
 } from '../lib/workload/stateless/filemanager/deploy/lib/filemanager';
-import { IcaEventPipeStackProps } from '../lib/workload/stateful/ica_event_pipe/stack';
+import { IcaEventPipeStackProps } from '../lib/workload/stateful/stacks/ica-event-pipe/stack';
+import { StatefulStackCollectionProps } from '../lib/workload/stateful/statefulStackCollectionClass';
+import { VpcLookupOptions } from 'aws-cdk-lib/aws-ec2';
+
+const region = 'ap-southeast-2';
 
 // upstream infra: vpc
 const vpcName = 'main-vpc';
 const vpcStackName = 'networking';
-const vpcProps = {
+const vpcProps: VpcLookupOptions = {
   vpcName: vpcName,
   tags: {
     Stack: vpcStackName,
@@ -26,7 +29,7 @@ const cognitoPortalAppClientIdParameterName = '/data_portal/client/data2/cog_app
 
 const regName = 'OrcaBusSchemaRegistry';
 const eventBusName = 'OrcaBusMain';
-const lambdaSecurityGroupName = 'OrcaBusLambdaSecurityGroup';
+const computeSecurityGroupName = 'OrcaBusSharedComputeSecurityGroup';
 const dbClusterIdentifier = 'orcabus-db';
 const dbClusterResourceIdParameterName = '/orcabus/db-cluster-resource-id';
 const dbClusterEndpointHostParameterName = '/orcabus/db-cluster-endpoint-host';
@@ -76,9 +79,8 @@ const orcaBusStatefulConfig = {
     clusterEndpointHostParameterName: dbClusterEndpointHostParameterName,
     secretRotationSchedule: Duration.days(7),
   },
-  securityGroupProps: {
-    securityGroupName: lambdaSecurityGroupName,
-    securityGroupDescription: 'allow within same SecurityGroup and rds SG',
+  computeProps: {
+    securityGroupName: computeSecurityGroupName,
   },
   icaEventPipeProps: icaEventPipeProps,
   tokenServiceProps: {
@@ -109,7 +111,7 @@ const orcaBusStatelessConfig = {
     ],
   },
   eventBusName: eventBusName,
-  lambdaSecurityGroupName: lambdaSecurityGroupName,
+  computeSecurityGroupName: computeSecurityGroupName,
   rdsMasterSecretName: rdsMasterSecretName,
   postgresManagerConfig: {
     masterSecretName: rdsMasterSecretName,
@@ -156,9 +158,10 @@ const filemanagerConfig = (bucket: string): FilemanagerConfig => {
 
 interface EnvironmentConfig {
   name: string;
+  region: string;
   accountId: string;
   stackProps: {
-    orcaBusStatefulConfig: OrcaBusStatefulConfig;
+    statefulConfig: StatefulStackCollectionProps;
     orcaBusStatelessConfig: OrcaBusStatelessConfig;
   };
 }
@@ -182,30 +185,28 @@ export const getEnvironmentConfig = (
     case 'beta':
       config = {
         name: 'beta',
+        region,
         accountId: '843407916570', // umccr_development
         stackProps: {
-          orcaBusStatefulConfig: {
-            schemaRegistryProps: {
-              ...orcaBusStatefulConfig.schemaRegistryProps,
-            },
-            eventBusProps: {
-              ...orcaBusStatefulConfig.eventBusProps,
-            },
-            databaseProps: {
-              ...orcaBusStatefulConfig.databaseProps,
-              numberOfInstance: 1,
-              minACU: 0.5,
-              maxACU: 16,
-              enhancedMonitoringInterval: Duration.seconds(60),
-              enablePerformanceInsights: true,
-              removalPolicy: RemovalPolicy.DESTROY,
+          statefulConfig: {
+            sharedStackProps: {
+              vpcProps,
+              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
+              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              databaseProps: {
+                ...orcaBusStatefulConfig.databaseProps,
+                numberOfInstance: 1,
+                minACU: 0.5,
+                maxACU: 16,
+                enhancedMonitoringInterval: Duration.seconds(60),
+                enablePerformanceInsights: true,
+                removalPolicy: RemovalPolicy.DESTROY,
+              },
+              computeProps: orcaBusStatefulConfig.computeProps,
+              eventSourceProps: eventSourceConfig(devBucket),
             },
-            securityGroupProps: {
-              ...orcaBusStatefulConfig.securityGroupProps,
-            },
-            eventSourceProps: eventSourceConfig(devBucket),
-            icaEventPipeProps: orcaBusStatefulConfig.icaEventPipeProps,
-            tokenServiceProps: { ...orcaBusStatefulConfig.tokenServiceProps },
+            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
           },
           orcaBusStatelessConfig: {
             ...orcaBusStatelessConfig,
@@ -218,30 +219,28 @@ export const getEnvironmentConfig = (
     case 'gamma':
       config = {
         name: 'gamma',
+        region,
         accountId: '455634345446', // umccr_staging
         stackProps: {
-          orcaBusStatefulConfig: {
-            schemaRegistryProps: {
-              ...orcaBusStatefulConfig.schemaRegistryProps,
-            },
-            eventBusProps: {
-              ...orcaBusStatefulConfig.eventBusProps,
+          statefulConfig: {
+            sharedStackProps: {
+              vpcProps,
+              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
+              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              databaseProps: {
+                ...orcaBusStatefulConfig.databaseProps,
+                numberOfInstance: 1,
+                minACU: 0.5,
+                maxACU: 16,
+                enhancedMonitoringInterval: Duration.seconds(60),
+                enablePerformanceInsights: true,
+                removalPolicy: RemovalPolicy.DESTROY,
+              },
+              computeProps: orcaBusStatefulConfig.computeProps,
+              eventSourceProps: eventSourceConfig(stgBucket),
             },
-            databaseProps: {
-              ...orcaBusStatefulConfig.databaseProps,
-              numberOfInstance: 1,
-              minACU: 0.5,
-              maxACU: 16,
-              enhancedMonitoringInterval: Duration.seconds(60),
-              enablePerformanceInsights: true,
-              removalPolicy: RemovalPolicy.DESTROY,
-            },
-            securityGroupProps: {
-              ...orcaBusStatefulConfig.securityGroupProps,
-            },
-            eventSourceProps: eventSourceConfig(stgBucket),
-            icaEventPipeProps: orcaBusStatefulConfig.icaEventPipeProps,
-            tokenServiceProps: { ...orcaBusStatefulConfig.tokenServiceProps },
+            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
           },
           orcaBusStatelessConfig: {
             ...orcaBusStatelessConfig,
@@ -254,28 +253,26 @@ export const getEnvironmentConfig = (
     case 'prod':
       config = {
         name: 'prod',
+        region,
         accountId: '472057503814', // umccr_production
         stackProps: {
-          orcaBusStatefulConfig: {
-            schemaRegistryProps: {
-              ...orcaBusStatefulConfig.schemaRegistryProps,
-            },
-            eventBusProps: {
-              ...orcaBusStatefulConfig.eventBusProps,
-            },
-            databaseProps: {
-              ...orcaBusStatefulConfig.databaseProps,
-              numberOfInstance: 1,
-              minACU: 0.5,
-              maxACU: 16,
-              removalPolicy: RemovalPolicy.RETAIN,
-            },
-            securityGroupProps: {
-              ...orcaBusStatefulConfig.securityGroupProps,
+          statefulConfig: {
+            sharedStackProps: {
+              vpcProps,
+              schemaRegistryProps: orcaBusStatefulConfig.schemaRegistryProps,
+              eventBusProps: orcaBusStatefulConfig.eventBusProps,
+              databaseProps: {
+                ...orcaBusStatefulConfig.databaseProps,
+                numberOfInstance: 1,
+                minACU: 0.5,
+                maxACU: 16,
+                removalPolicy: RemovalPolicy.RETAIN,
+              },
+              computeProps: orcaBusStatefulConfig.computeProps,
+              eventSourceProps: eventSourceConfig(prodBucket),
             },
-            eventSourceProps: eventSourceConfig(prodBucket),
-            icaEventPipeProps: orcaBusStatefulConfig.icaEventPipeProps,
-            tokenServiceProps: { ...orcaBusStatefulConfig.tokenServiceProps },
+            tokenServiceStackProps: orcaBusStatefulConfig.tokenServiceProps,
+            icaEventPipeStackProps: orcaBusStatefulConfig.icaEventPipeProps,
           },
           orcaBusStatelessConfig: {
             ...orcaBusStatelessConfig,
@@ -286,7 +283,7 @@ export const getEnvironmentConfig = (
       break;
   }
 
-  validateSecretName(config.stackProps.orcaBusStatefulConfig.databaseProps.masterSecretName);
+  // validateSecretName(config.stackProps.orcaBusStatefulConfig.databaseProps.masterSecretName);
 
   return config;
 };

docs/developer/MICROSERVICE.md

@@ -76,7 +76,7 @@ Most probably you microservice stack should only create new stateless resources
 be provisioned from the shared stateful stack. For example, your application may need an RDS cluster for its database,
 but the shared stack has an existing RDS cluster that is intended to be used across microservices.
 
-See [SHARED_RESOURCES.md](./SHARED_RESOURCES.md) for more shared resources detail.
+See [README.md](../../lib/workload/stateful/stacks/shared/README.md) in the stateful shared stack for more detail.
 
 Useful resources:
 

docs/developer/SHARED_RESOURCES.md

@@ -1,44 +1,7 @@
 # Shared Resources
 
-In the stateful world of the OrcaBus we will be sharing some resources so it could be used across microservices.
-These resources will be deployed into a stack and will go under the CDK stateful app.
-
-These stateful resources usually have a unique name that could act as an Id for the resource. The unique name will be
-defined at the CDK config file where it could be passed in both stateful and stateless stack. The stateless stack can
-use the resource by the CDK lookup.
-
-
-## Shared
-### Database
-
-An Amazon Aurora Serverless PostgreSQL is provisioned to be used across microservices.
-
-A security group is created and available for lookup that could be attached to your compute which allow traffic to the
-RDS cluster. The security group name is in the CDK config that your microservice could pass this in as one of the stack props.
-
-Each RDS cluster could contain multiple databases and each microservice is expected to to create their own database and
-role to be used in their application. There is a microservice called  `PostgresManager` that specifically handle this administrative
-task on PostgreSQL.
-
-RDS IAM is enabled for the cluster, therefore is encouraged to used rather than relying on username-password approach to login to your
-database. You could choose the type of the authentication upon creating a role at the RDS when using the `PostgresManager`.
-
-Please check the: [PostgresManager](../../lib/workload/stateless/postgres_manager/README.md)
-
-
-### Event Source
-
-### Eventbridge
-
 ## ICA Event Pipe
 
 This stack creates the necessary infrastructure to allow external (ICA) events to flow onto our internal Event Bus.
 
 Details see [here](../../lib/workload/stateful/ica_event_pipe/README.md)
-
-
-## Schema Registry
-
-## SecurityGroup
-
-## VPC