From 40c265b8b26f02e2c9b797a12582614cfd18a4a9 Mon Sep 17 00:00:00 2001 From: Bob van den Hoogen Date: Mon, 23 Oct 2023 16:44:09 +0200 Subject: [PATCH 1/6] chore: add proto validations --- .gitignore | 2 - Makefile | 32 -------- protos/README.md | 3 + protos/buf.lock | 8 +- protos/buf.yaml | 5 +- .../v1alpha/data_policies_v1alpha.proto | 5 +- .../v1alpha/entities_v1alpha.proto | 37 ++++----- protos/setup-ide-protobuf-plugins.sh | 81 ------------------- 8 files changed, 31 insertions(+), 142 deletions(-) create mode 100644 protos/README.md delete mode 100755 protos/setup-ide-protobuf-plugins.sh diff --git a/.gitignore b/.gitignore index 82f0dbdb..6962f7c0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,4 @@ out/ /app/build/ output/ -.common-protos - app/src/main/resources/application-local.yaml diff --git a/Makefile b/Makefile index d9770a0a..5c0cae46 100644 --- a/Makefile +++ b/Makefile @@ -1,41 +1,9 @@ .PHONY: clean-common-protos -common_protos := ${CURDIR}/.common-protos - grpc_version := 1.50.0 protobuf_version := 3.21.9 -google_common_protos_version := 2.10.0 git_branch := $(shell git rev-parse --abbrev-ref HEAD) -# google/protobuf dependencies (predefined Protos for e.g. Timestamp, Duration, etc) -${common_protos}/protobuf-java.jar: - curl "https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/${protobuf_version}/protobuf-java-${protobuf_version}.jar" --create-dirs -o "${common_protos}/protobuf-java.jar" - -# google/api dependencies (Common Google Protos, such as field_behavior) -${common_protos}/proto-google-common-protos.jar: - curl "https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-common-protos/${google_common_protos_version}/proto-google-common-protos-${google_common_protos_version}.jar" --create-dirs -o "${common_protos}/proto-google-common-protos.jar" - -${common_protos}: ${common_protos}/proto-google-common-protos.jar ${common_protos}/protobuf-java.jar ${common_protos}/validate/validate.proto - -clean-common-protos: - rm -rf ${common_protos} - -${common_protos}/google/protobuf: ${common_protos}/protobuf-java.jar - unzip -d ${common_protos} $< "google/**/*.proto" - -${common_protos}/google/api: ${common_protos}/proto-google-common-protos.jar - unzip -d ${common_protos} $< "google/**/*.proto" - -${common_protos}/validate/validate.proto: - curl "https://raw.githubusercontent.com/bufbuild/protoc-gen-validate/v0.9.1/validate/validate.proto" --create-dirs -o "${common_protos}/validate/validate.proto" - -# To ensure that we use the same Google Common Protobuf files in all languages, we extract them from the jar -default-google-dependencies: ${common_protos}/google/protobuf ${common_protos}/google/api -protoc-gen-validate-dependency: ${common_protos}/validate/validate.proto - -intellij: ${common_protos} - ./protos/setup-ide-protobuf-plugins.sh - buf-publish-current-branch: [[ "$$OSTYPE" == "darwin"* ]] && SED=gsed || SED=sed && \ commit_hash=$$(cd protos && buf push --branch "${git_branch}") && \ diff --git a/protos/README.md b/protos/README.md new file mode 100644 index 00000000..b0cd79e5 --- /dev/null +++ b/protos/README.md @@ -0,0 +1,3 @@ +1. Install the `Buf for Protocol Buffers` and the bundled `gRPC` plugins for Intellij. +2. Configure your buf path in the plugin settings (e.g. `$ which buf`) +3. Update buf with `buf mod update` \ No newline at end of file diff --git a/protos/buf.lock b/protos/buf.lock index 7a138187..f39d844d 100644 --- a/protos/buf.lock +++ b/protos/buf.lock @@ -2,10 +2,10 @@ version: v1 deps: - remote: buf.build - owner: envoyproxy - repository: protoc-gen-validate - commit: eac44469a7af47e7839a7f1f3d7ac004 - digest: shake256:0feabcde01b6b11e3c75a5e3f807968d5995626546f39c37e5d4205892b3a59cced0ed83b35a2eb9e6dddd3309660ad46b737c9dcd224b425de0a6654ce04417 + owner: bufbuild + repository: protovalidate + commit: 0de7443d03cf41228f8a9790b12b417e + digest: shake256:3c0676a73cef06439c107cb9560627354815adbc254976f807d645de7e2c1bf19d0438d5d56d5bc92465377e0d9315951e986fc6ced2871e450534b2b8c953b0 - remote: buf.build owner: googleapis repository: googleapis diff --git a/protos/buf.yaml b/protos/buf.yaml index b42e6b9e..dc92d5e7 100644 --- a/protos/buf.yaml +++ b/protos/buf.yaml @@ -1,9 +1,8 @@ version: v1 -name: buf.build/getstrm/daps +name: buf.build/getstrm/pace deps: - buf.build/googleapis/googleapis - # FIXME once an official repo is available for bufbuild/protoc-gen-validate, we should change to that - - buf.build/envoyproxy/protoc-gen-validate + - buf.build/bufbuild/protovalidate breaking: use: - FILE diff --git a/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto b/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto index 412593de..96d44098 100644 --- a/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto +++ b/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto @@ -4,6 +4,7 @@ package getstrm.api.data_policies.v1alpha; import "getstrm/api/data_policies/v1alpha/entities_v1alpha.proto"; import "google/api/annotations.proto"; +import "buf/validate/validate.proto"; option java_package = "com.getstrm.api.data_policies.v1alpha"; option java_multiple_files = true; @@ -124,7 +125,7 @@ message ListProcessingPlatformGroupsResponse { message GetProcessingPlatformBarePolicyRequest { DataPolicy.ProcessingPlatform platform = 1; - string table = 2; + string table = 2 [(buf.validate.field) = {string: {min_len: 1}, ignore_empty: false}]; } message GetProcessingPlatformBarePolicyResponse { @@ -133,7 +134,7 @@ message GetProcessingPlatformBarePolicyResponse { message GetDataPolicyRequest { - string id = 1; + string id = 1 [(buf.validate.field).string.uuid = true]; } message GetDataPolicyResponse { diff --git a/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto b/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto index cdbcc571..e04258be 100644 --- a/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto +++ b/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto @@ -2,20 +2,21 @@ syntax = "proto3"; package getstrm.api.data_policies.v1alpha; +import "buf/validate/validate.proto"; import "google/protobuf/timestamp.proto"; message DataPolicy { - string id = 1; + string id = 1 [(buf.validate.field).string.uuid = true]; Info info = 2; Source source = 3; ProcessingPlatform platform = 4; repeated RuleSet rule_sets = 5; message Info { - string title = 1; - string description = 2; - string version = 3; - string organization_id = 4; + string title = 1 [(buf.validate.field).string.min_len = 3]; + string description = 2 [(buf.validate.field).ignore_empty = true]; + string version = 3 [(buf.validate.field).string.pattern = "^\\d+\\.\\d+\\.\\d+$"]; + string organization_id = 4 [(buf.validate.field) = {string: {uuid: true}, ignore_empty: true}]; google.protobuf.Timestamp create_time = 5; google.protobuf.Timestamp update_time = 6; } @@ -30,7 +31,7 @@ message DataPolicy { // Last condition in the list must have 0 principals, as this acts as the default / else condition. repeated Condition conditions = 3; message Condition { - repeated string principals = 1; + repeated string principals = 1 [(buf.validate.field).repeated = { items: { string: {min_len: 1} }, min_items: 0 }]; string condition = 2; } } @@ -40,7 +41,7 @@ message DataPolicy { // Last condition in the list must have 0 principals, as this acts as the default / else condition. repeated Transform transforms = 2; message Transform { - repeated string principals = 1; + repeated string principals = 1 [(buf.validate.field).repeated = { items: { string: {min_len: 1} } }]; oneof transform { RegexReplace regex = 2; bool identity = 3; @@ -70,7 +71,7 @@ message DataPolicy { message ProcessingPlatform { PlatformType platform_type = 1; - string id = 2; + string id = 2 [(buf.validate.field).string.min_len = 1]; enum PlatformType { PLATFORM_TYPE_UNSPECIFIED = 0; DATABRICKS = 1; @@ -81,7 +82,7 @@ message DataPolicy { message Target { TargetType type = 1; - string fullname = 2; + string fullname = 2 [(buf.validate.field).string.min_len = 1]; enum TargetType { TARGET_TYPE_UNSPECIFIED = 0; DYNAMIC_VIEW = 1; @@ -90,7 +91,7 @@ message DataPolicy { message Source { // The ref is an identifier for the source, unique at the source platform level. - string ref = 1; + string ref = 1 [(buf.validate.field).string.min_len = 3]; Type type = 2; // Note: we are currently not using the string-based spec after all. Using the Attributes instead improves // testability and should make it easier for the frontend to render the policy. Otherwise the frontend @@ -113,7 +114,7 @@ message DataPolicy { message Attribute { // Instead of using "nodes", nesting can be expressed by specifying multiple path components. This is similar to BigQuery. - repeated string path_components = 1; + repeated string path_components = 1 [(buf.validate.field).repeated = { items: { string: {min_len: 1} } }]; // This is the "native" type, originating from the source platform. string type = 2; bool required = 3; @@ -123,7 +124,7 @@ message DataPolicy { message DataCatalog { - string id = 1; + string id = 1 [(buf.validate.field) = {ignore_empty: false, string: {min_len: 1}}]; Type type = 2; repeated DataBase databases = 3; enum Type { @@ -134,23 +135,23 @@ message DataCatalog { } message DataBase { - string id = 1; + string id = 1 [(buf.validate.field) = {ignore_empty: false, string: {min_len: 1}}]; string type = 2; - string display_name = 3; + string display_name = 3 [(buf.validate.field) = {ignore_empty: false, string: {min_len: 1}}]; DataCatalog catalog = 4; repeated Schema schemas = 5; } message Schema { - string id = 1; - string name = 2; + string id = 1 [(buf.validate.field) = {ignore_empty: false, string: {min_len: 1}}]; + string name = 2 [(buf.validate.field) = {ignore_empty: false, string: {min_len: 1}}]; DataBase database = 3; repeated Table tables = 4; } message Table { - string id = 1; - string name = 2; + string id = 1 [(buf.validate.field) = {ignore_empty: false, string: {min_len: 1}}]; + string name = 2 [(buf.validate.field) = {ignore_empty: false, string: {min_len: 1}}]; Schema schema = 3; } } diff --git a/protos/setup-ide-protobuf-plugins.sh b/protos/setup-ide-protobuf-plugins.sh deleted file mode 100755 index 5246c69b..00000000 --- a/protos/setup-ide-protobuf-plugins.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -REPO_ROOT_DIR="$( cd "$( dirname "$0" )/.." && pwd )" -IDEA_DIR="$REPO_ROOT_DIR/.idea" - -# Settings for Official Jetbrains Protocol Buffers plugin -cat << EOF > "$IDEA_DIR/protoeditor.xml" - - - - - - -EOF - -# Settings for HIGAN IntelliJ Protobuf Plugin -cat << EOF > "$IDEA_DIR/protobuf.xml" - - - - - - - - - - - -EOF - -# Settings to mark protos directory as sources -IML_FILE=$(find "$IDEA_DIR" -name '*.iml') - -if [ -f "$IML_FILE" ]; then - if [[ "$OSTYPE" == "darwin"* ]]; then - SED=gsed - else - SED=sed - fi - - $SED -i 's|||g' "$IML_FILE" -else -cat <<-EOF > "$IDEA_DIR/pace.iml" - - - - - - - - - - - - - - - -EOF -fi From e44f462c7d359dbf8e9aed1abf9b09c1ac0654c3 Mon Sep 17 00:00:00 2001 From: Bob van den Hoogen Date: Mon, 23 Oct 2023 18:09:24 +0200 Subject: [PATCH 2/6] fix: create new buf repo for pace APIs; add http endpoints for all APIs --- app/build.gradle.kts | 10 +-- .../com/getstrm/pace/api/DataPolicyApi.kt | 4 +- .../service/ProcessingPlatformsService.kt | 10 +-- gradle.properties | 2 +- protos/buf.yaml | 2 +- .../v1alpha/data_policies_v1alpha.proto | 64 +++++++++++++------ 6 files changed, 60 insertions(+), 32 deletions(-) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 7aac037d..fce671de 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -56,9 +56,9 @@ dependencies { implementation(enforcedPlatform("com.google.cloud:libraries-bom:26.24.0")) implementation("com.google.cloud:google-cloud-bigquery") - implementation("build.buf.gen:getstrm_daps_grpc_java:1.58.0.1.$generatedBufDependencyVersion") - implementation("build.buf.gen:getstrm_daps_grpc_kotlin:1.3.1.1.$generatedBufDependencyVersion") - implementation("build.buf.gen:getstrm_daps_protocolbuffers_java:24.4.0.1.$generatedBufDependencyVersion") + implementation("build.buf.gen:getstrm_pace_grpc_java:1.58.0.1.$generatedBufDependencyVersion") + implementation("build.buf.gen:getstrm_pace_grpc_kotlin:1.3.1.1.$generatedBufDependencyVersion") + implementation("build.buf.gen:getstrm_pace_protocolbuffers_java:24.4.0.1.$generatedBufDependencyVersion") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml") @@ -116,9 +116,9 @@ kotlin { } tasks.named("bootJar") { - mainClass = "com.getstrm.daps.DataPolicyServiceApplicationKt" + mainClass = "com.getstrm.pace.DataPolicyServiceApplicationKt" manifest { - attributes["Implementation-Title"] = "Data Policy Service" + attributes["Implementation-Title"] = "Policy and Contract Engine" attributes["Implementation-Version"] = version } } diff --git a/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt b/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt index 0734dcfb..0d44eedf 100644 --- a/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt +++ b/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt @@ -25,7 +25,7 @@ class DataPolicyApi( override suspend fun getDataPolicy(request: GetDataPolicyRequest): GetDataPolicyResponse { return GetDataPolicyResponse.newBuilder() - .setDataPolicy(dataPolicyService.getLatestDataPolicy(request.id)) + .setDataPolicy(dataPolicyService.getLatestDataPolicy(request.dataPolicyId)) .build() } @@ -51,6 +51,6 @@ class DataPolicyApi( override suspend fun getProcessingPlatformBarePolicy(request: GetProcessingPlatformBarePolicyRequest): GetProcessingPlatformBarePolicyResponse = GetProcessingPlatformBarePolicyResponse.newBuilder() - .setDataPolicy(processingPlatformsService.createBarePolicy(request.platform, request.table)) + .setDataPolicy(processingPlatformsService.createBarePolicy(request.platformId, request.tableId)) .build() } diff --git a/app/src/main/kotlin/com/getstrm/pace/service/ProcessingPlatformsService.kt b/app/src/main/kotlin/com/getstrm/pace/service/ProcessingPlatformsService.kt index 43c8c6c7..a192f927 100644 --- a/app/src/main/kotlin/com/getstrm/pace/service/ProcessingPlatformsService.kt +++ b/app/src/main/kotlin/com/getstrm/pace/service/ProcessingPlatformsService.kt @@ -39,17 +39,17 @@ class ProcessingPlatformsService( } suspend fun listProcessingPlatformTables(request: ListProcessingPlatformTablesRequest): List = - (platforms[request.platform.id] ?: throw ProcessingPlatformNotFoundException(request.platform.id)).listTables() + (platforms[request.platformId] ?: throw ProcessingPlatformNotFoundException(request.platformId)).listTables() suspend fun listProcessingPlatformGroups(request: ListProcessingPlatformGroupsRequest): List = - (platforms[request.platform.id] ?: throw ProcessingPlatformNotFoundException(request.platform.id)).listGroups() + (platforms[request.platformId] ?: throw ProcessingPlatformNotFoundException(request.platformId)).listGroups() - suspend fun createBarePolicy(platform: DataPolicy.ProcessingPlatform?, tableName: String): DataPolicy { + suspend fun createBarePolicy(platformId: String, tableName: String): DataPolicy { val processingPlatformInterface = - platforms[platform!!.id] ?: throw ProcessingPlatformNotFoundException(platform.id) + platforms[platformId] ?: throw ProcessingPlatformNotFoundException(platformId) val table = processingPlatformInterface.createTable(tableName) return table.toDataPolicy( - DataPolicy.ProcessingPlatform.newBuilder().setId(platform.id) + DataPolicy.ProcessingPlatform.newBuilder().setId(platformId) .setPlatformType(processingPlatformInterface.type).build() ) } diff --git a/gradle.properties b/gradle.properties index 376e4b9a..c6514513 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,4 +8,4 @@ jooqVersion=3.18.7 # TODO verify the tag is correct dockertag = ghcr.io/getstrm/pace:latest -generatedBufDependencyVersion=00000000000000.0f57123e7845 +generatedBufDependencyVersion=00000000000000.7d5c0b141bd9 diff --git a/protos/buf.yaml b/protos/buf.yaml index b42e6b9e..2200df94 100644 --- a/protos/buf.yaml +++ b/protos/buf.yaml @@ -1,5 +1,5 @@ version: v1 -name: buf.build/getstrm/daps +name: buf.build/getstrm/pace deps: - buf.build/googleapis/googleapis # FIXME once an official repo is available for bufbuild/protoc-gen-validate, we should change to that diff --git a/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto b/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto index 412593de..69388acc 100644 --- a/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto +++ b/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto @@ -23,7 +23,7 @@ service DataPolicyService { // returns latest policy for an id rpc GetDataPolicy (GetDataPolicyRequest) returns (GetDataPolicyResponse) { option (google.api.http) = { - get: "/data-policy/{id}" + get: "/data-policy/{data_policy_id}" }; }; // return configured processing platforms in DPS @@ -35,31 +35,56 @@ service DataPolicyService { // return table names as known by the platform rpc ListProcessingPlatformTables (ListProcessingPlatformTablesRequest) returns (ListProcessingPlatformTablesResponse) { option (google.api.http) = { - get: "/processing-platforms/{platform}/tables" + get: "/processing-platforms/{platform_id}/tables" }; }; // return groups as known by the platform rpc ListProcessingPlatformGroups (ListProcessingPlatformGroupsRequest) returns (ListProcessingPlatformGroupsResponse) { option (google.api.http) = { - get: "/processing-platforms/{platform}/groups" + get: "/processing-platforms/{platform_id}/groups" }; }; // return a data-policy without rules sets as built from the table description on the platform rpc GetProcessingPlatformBarePolicy (GetProcessingPlatformBarePolicyRequest) returns (GetProcessingPlatformBarePolicyResponse) { option (google.api.http) = { - get: "/processing-platforms/{platform}/tables/{table}/bare-policy" + get: "/processing-platforms/{platform_id}/tables/{table_id}/bare-policy" }; }; - rpc ListCatalogs (ListCatalogsRequest) returns (ListCatalogsResponse); - rpc ListDatabases (ListDatabasesRequest) returns (ListDatabasesResponse); - rpc ListSchemas (ListSchemasRequest) returns (ListSchemasResponse); - rpc ListTables (ListTablesRequest) returns (ListTablesResponse); + + rpc ListCatalogs (ListCatalogsRequest) returns (ListCatalogsResponse) { + option (google.api.http) = { + get: "/catalogs" + }; + }; + rpc ListDatabases (ListDatabasesRequest) returns (ListDatabasesResponse) { + option (google.api.http) = { + get: "/catalogs/{catalog_id}/databases" + }; + } + rpc ListSchemas (ListSchemasRequest) returns (ListSchemasResponse) { + option (google.api.http) = { + get: "/catalogs/{catalog_id}/databases/{database_id}/schemas" + }; + } + rpc ListTables (ListTablesRequest) returns (ListTablesResponse) { + option (google.api.http) = { + get: "/catalogs/{catalog_id}/databases/{database_id}/schemas/{schema_id}/tables" + }; + }; + // return a data-policy without rules sets as built from the table description on the platform - rpc GetCatalogBarePolicy (GetCatalogBarePolicyRequest) returns (GetCatalogBarePolicyResponse); + rpc GetCatalogBarePolicy (GetCatalogBarePolicyRequest) returns (GetCatalogBarePolicyResponse) { + option (google.api.http) = { + get: "/catalogs/{catalog_id}/databases/{database_id}/schemas/{schema_id}/tables/{table_id}/bare-policy" + }; + }; } message GetCatalogBarePolicyRequest { - DataCatalog.Table table = 1; + string catalog_id = 1; + string database_id = 2; + string schema_id = 3; + string table_id = 4; } message GetCatalogBarePolicyResponse { @@ -74,7 +99,7 @@ message ListCatalogsResponse { } message ListDatabasesRequest { - DataCatalog catalog = 1; + string catalog_id = 1; } message ListDatabasesResponse { @@ -82,7 +107,8 @@ message ListDatabasesResponse { } message ListSchemasRequest { - DataCatalog.DataBase database = 1; + string catalog_id = 1; + string database_id = 2; } // (-- api-linter: core::0158::response-plural-first-field=disabled @@ -92,7 +118,9 @@ message ListSchemasResponse { } message ListTablesRequest { - DataCatalog.Schema schema = 1; + string catalog_id = 1; + string database_id = 2; + string schema_id = 3; } message ListTablesResponse { @@ -107,7 +135,7 @@ message ListProcessingPlatformsResponse { } message ListProcessingPlatformTablesRequest { - DataPolicy.ProcessingPlatform platform = 1; + string platform_id = 1; } message ListProcessingPlatformTablesResponse { @@ -115,7 +143,7 @@ message ListProcessingPlatformTablesResponse { } message ListProcessingPlatformGroupsRequest { - DataPolicy.ProcessingPlatform platform = 1; + string platform_id = 1; } message ListProcessingPlatformGroupsResponse { @@ -123,8 +151,8 @@ message ListProcessingPlatformGroupsResponse { } message GetProcessingPlatformBarePolicyRequest { - DataPolicy.ProcessingPlatform platform = 1; - string table = 2; + string platform_id = 1; + string table_id = 2; } message GetProcessingPlatformBarePolicyResponse { @@ -133,7 +161,7 @@ message GetProcessingPlatformBarePolicyResponse { message GetDataPolicyRequest { - string id = 1; + string data_policy_id = 1; } message GetDataPolicyResponse { From 895ec440e0e860c2c05e165eefc0d0127ca32fed Mon Sep 17 00:00:00 2001 From: Ivan Plantevin Date: Wed, 25 Oct 2023 13:36:57 +0200 Subject: [PATCH 3/6] feat(strm-2703): use optional in proto messages for primitive types --- gradle.properties | 2 +- .../data_policies/v1alpha/entities_v1alpha.proto | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/gradle.properties b/gradle.properties index a0ab58a2..594bdf33 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,4 +8,4 @@ jooqVersion=3.18.7 # TODO verify the tag is correct dockertag = ghcr.io/getstrm/pace:latest -generatedBufDependencyVersion=00000000000000.609525407a2f +generatedBufDependencyVersion=00000000000000.4c112bea69d1 diff --git a/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto b/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto index e589cb7c..a5e6c098 100644 --- a/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto +++ b/protos/getstrm/api/data_policies/v1alpha/entities_v1alpha.proto @@ -13,9 +13,9 @@ message DataPolicy { message Info { string title = 1; - string description = 2; + optional string description = 2; string version = 3; - string organization_id = 4; + string context = 4; google.protobuf.Timestamp create_time = 5; google.protobuf.Timestamp update_time = 6; repeated string tags = 7; @@ -53,14 +53,15 @@ message DataPolicy { string statement = 1; } message Hash { - int64 seed = 1; + optional int64 seed = 1; } message FixedValue { string value = 1; } + // Todo: we probably should create a separate RegexExtract, since a RegexReplace with empty replacement is functionally different. message RegexReplace { string regex = 1; - string replacement = 2; + optional string replacement = 2; } message Nullify { } @@ -95,6 +96,7 @@ message DataPolicy { // Note: we are currently not using the string-based spec after all. Using the Attributes instead improves // testability and should make it easier for the frontend to render the policy. Otherwise the frontend // would have to parse the string-based spec. + // Todo: remove the string-based spec for now? string spec = 3; // attributes are leading. @@ -119,8 +121,6 @@ message DataPolicy { string type = 2; bool required = 3; repeated string tags = 4; - - // TODO add repeated? } } @@ -137,10 +137,12 @@ message DataCatalog { DATAHUB = 3; } + // Todo: apart from the discussion on hierarchy vs. flat, Database is written without uppercase B. message DataBase { string id = 1; string type = 2; string display_name = 3; + // Todo: complete parent/child objects instead of ids can sometimes result in some cyclic issues. DataCatalog catalog = 4; repeated Schema schemas = 5; repeated string tags = 6; From ce057e0cc63dd535ca6e872fcb2e39855b2a98da Mon Sep 17 00:00:00 2001 From: Ivan Plantevin Date: Wed, 25 Oct 2023 13:38:51 +0200 Subject: [PATCH 4/6] fix(strm-2707): make data-policy REST path plural --- .../api/data_policies/v1alpha/data_policies_v1alpha.proto | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto b/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto index 69388acc..dbcd02bc 100644 --- a/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto +++ b/protos/getstrm/api/data_policies/v1alpha/data_policies_v1alpha.proto @@ -16,14 +16,14 @@ service DataPolicyService { }; rpc UpsertDataPolicy (UpsertDataPolicyRequest) returns (UpsertDataPolicyResponse) { option (google.api.http) = { - post: "/data-policy" + post: "/data-policies" body: "*" }; }; // returns latest policy for an id rpc GetDataPolicy (GetDataPolicyRequest) returns (GetDataPolicyResponse) { option (google.api.http) = { - get: "/data-policy/{data_policy_id}" + get: "/data-policies/{data_policy_id}" }; }; // return configured processing platforms in DPS @@ -186,5 +186,3 @@ message ListDataPoliciesRequest { message ListDataPoliciesResponse { repeated DataPolicy data_policies = 1; } - - From f77762eaaf165fb34539c579852bc1077c8a6aac Mon Sep 17 00:00:00 2001 From: Ivan Plantevin Date: Wed, 25 Oct 2023 14:59:02 +0200 Subject: [PATCH 5/6] fix(strm-2707): modify policy api to proto call changes --- .../com/getstrm/pace/api/DataPolicyApi.kt | 39 ++++++++++++--- .../getstrm/pace/service/CatalogService.kt | 47 ++++++++++++------- gradle.properties | 2 +- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt b/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt index ed7569b4..fccb0323 100644 --- a/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt +++ b/app/src/main/kotlin/com/getstrm/pace/api/DataPolicyApi.kt @@ -1,6 +1,31 @@ package com.getstrm.pace.api -import build.buf.gen.getstrm.api.data_policies.v1alpha.* +import build.buf.gen.getstrm.api.data_policies.v1alpha.DataPolicy +import build.buf.gen.getstrm.api.data_policies.v1alpha.DataPolicyServiceGrpcKt +import build.buf.gen.getstrm.api.data_policies.v1alpha.GetCatalogBarePolicyRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.GetCatalogBarePolicyResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.GetDataPolicyRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.GetDataPolicyResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.GetProcessingPlatformBarePolicyRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.GetProcessingPlatformBarePolicyResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListCatalogsRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListCatalogsResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListDataPoliciesRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListDataPoliciesResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListDatabasesRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListDatabasesResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListProcessingPlatformGroupsRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListProcessingPlatformGroupsResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListProcessingPlatformTablesRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListProcessingPlatformTablesResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListProcessingPlatformsRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListProcessingPlatformsResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListSchemasRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListSchemasResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListTablesRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.ListTablesResponse +import build.buf.gen.getstrm.api.data_policies.v1alpha.UpsertDataPolicyRequest +import build.buf.gen.getstrm.api.data_policies.v1alpha.UpsertDataPolicyResponse import com.getstrm.pace.service.CatalogService import com.getstrm.pace.service.DataPolicyService import com.getstrm.pace.service.ProcessingPlatformsService @@ -53,7 +78,7 @@ class DataPolicyApi( override suspend fun getProcessingPlatformBarePolicy(request: GetProcessingPlatformBarePolicyRequest): GetProcessingPlatformBarePolicyResponse = GetProcessingPlatformBarePolicyResponse.newBuilder() - .setDataPolicy(processingPlatformsService.createBarePolicy(request.platformId, request.tableId)) + .setDataPolicy(processingPlatformsService.createBarePolicy(request.platformId, request.table)) .build() override suspend fun listCatalogs(request: ListCatalogsRequest): ListCatalogsResponse = @@ -62,26 +87,28 @@ class DataPolicyApi( .build() override suspend fun listDatabases(request: ListDatabasesRequest): ListDatabasesResponse { - val databases = catalogService.listDatabases(request.catalog) + val databases = catalogService.listDatabases(request.catalogId) return ListDatabasesResponse.newBuilder() .addAllDatabases(databases) .build() } override suspend fun listSchemas(request: ListSchemasRequest): ListSchemasResponse { - val schemas = catalogService.listSchemas(request.database) + val schemas = catalogService.listSchemas(request.catalogId, request.databaseId) return ListSchemasResponse.newBuilder() .addAllSchemas(schemas) .build() } override suspend fun listTables(request: ListTablesRequest): ListTablesResponse { - val tables = catalogService.listTables(request.schema) + val tables = catalogService.listTables(request.catalogId, request.databaseId, request.schemaId) return ListTablesResponse.newBuilder() .addAllTables(tables) .build() } override suspend fun getCatalogBarePolicy(request: GetCatalogBarePolicyRequest): GetCatalogBarePolicyResponse { - val dataPolicy: DataPolicy = catalogService.getBarePolicy(request.table) + val dataPolicy: DataPolicy = catalogService.getBarePolicy( + request.catalogId, request.databaseId, request.schemaId, request.tableId + ) return GetCatalogBarePolicyResponse.newBuilder() .setDataPolicy(dataPolicy) .build() diff --git a/app/src/main/kotlin/com/getstrm/pace/service/CatalogService.kt b/app/src/main/kotlin/com/getstrm/pace/service/CatalogService.kt index 07214b8c..b765d76e 100644 --- a/app/src/main/kotlin/com/getstrm/pace/service/CatalogService.kt +++ b/app/src/main/kotlin/com/getstrm/pace/service/CatalogService.kt @@ -40,24 +40,32 @@ class CatalogService( .build() } - suspend fun listDatabases(apiCatalog: ApiCatalog): List = - getCatalog(apiCatalog.id).listDatabases().map { it.apiDatabase } + suspend fun listDatabases(catalogId: String): List = + getCatalog(catalogId).listDatabases().map { it.apiDatabase } - suspend fun listSchemas(apiDatabase: ApiDatabase): List { - val catalog = getCatalog(apiDatabase.catalog.id) - val database = catalog.listDatabases().find { it.id == apiDatabase.id } - ?: throw CatalogDatabaseNotFoundException(apiDatabase.id) + suspend fun listSchemas(catalogId: String, databaseId: String): List { + val catalog = getCatalog(catalogId) + val database = catalog.listDatabases().firstOrNull { it.id == databaseId } + ?: throw CatalogDatabaseNotFoundException(databaseId) val schemas = database.getSchemas() return schemas.map { it.apiSchema } } - suspend fun listTables(apiSchema: ApiSchema): List = - getTablesInfo(apiSchema).map { it.apiTable } + suspend fun listTables( + catalogId: String, + databaseId: String, + schemaId: String, + ): List = getTablesInfo(catalogId, databaseId, schemaId).map { it.apiTable } - suspend fun getBarePolicy(apiTable: ApiTable): DataPolicy { - val tables = getTablesInfo(apiTable.schema) - val table = tables.find { it.id == apiTable.id } - ?: throw CatalogTableNotFoundException(apiTable.id) + suspend fun getBarePolicy( + catalogId: String, + databaseId: String, + schemaId: String, + tableId: String, + ): DataPolicy { + val tables = getTablesInfo(catalogId, databaseId, schemaId) + val table = tables.firstOrNull { it.id == tableId } + ?: throw CatalogTableNotFoundException(tableId) return table.getDataPolicy()!! } @@ -66,12 +74,17 @@ class CatalogService( * * @return dto object with all relevant info */ - private suspend fun getTablesInfo(apiSchema: ApiSchema): List { - val catalog = getCatalog(apiSchema.database.catalog.id) - val database = catalog.listDatabases().find { it.id == apiSchema.database.id } - ?: throw CatalogDatabaseNotFoundException(apiSchema.database.id) + private suspend fun getTablesInfo( + catalogId: String, + databaseId: String, + schemaId: String, + ): List { + val catalog = getCatalog(catalogId) + val database = catalog.listDatabases().firstOrNull { it.id == databaseId } + ?: throw CatalogDatabaseNotFoundException(databaseId) val schema = - database.getSchemas().find { it.id == apiSchema.id } ?: throw CatalogSchemaNotFoundException(apiSchema.id) + database.getSchemas().firstOrNull { it.id == schemaId } + ?: throw CatalogSchemaNotFoundException(schemaId) return schema.getTables() } diff --git a/gradle.properties b/gradle.properties index a0ab58a2..45eb036c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,4 +8,4 @@ jooqVersion=3.18.7 # TODO verify the tag is correct dockertag = ghcr.io/getstrm/pace:latest -generatedBufDependencyVersion=00000000000000.609525407a2f +generatedBufDependencyVersion=00000000000000.64c7ab603c22 From d6a288205a897e8a7f3751a26fb339b33a99771a Mon Sep 17 00:00:00 2001 From: Ivan Plantevin Date: Wed, 25 Oct 2023 16:32:14 +0200 Subject: [PATCH 6/6] fix(strm-2707): use latest buf generated api --- app/src/main/kotlin/com/getstrm/pace/dao/DataPolicyDao.kt | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/getstrm/pace/dao/DataPolicyDao.kt b/app/src/main/kotlin/com/getstrm/pace/dao/DataPolicyDao.kt index c19fbc9f..c3c4f6b7 100644 --- a/app/src/main/kotlin/com/getstrm/pace/dao/DataPolicyDao.kt +++ b/app/src/main/kotlin/com/getstrm/pace/dao/DataPolicyDao.kt @@ -36,7 +36,7 @@ class DataPolicyDao( dataPolicy.info.toBuilder() .setUpdateTime(updateTimestamp.toTimestamp()) .setCreateTime(oldPolicy?.info?.createTime ?: updateTimestamp.toTimestamp()) - .setOrganizationId(context) + .setContext(context) .build() ).build() diff --git a/gradle.properties b/gradle.properties index 45eb036c..5e29172c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -8,4 +8,4 @@ jooqVersion=3.18.7 # TODO verify the tag is correct dockertag = ghcr.io/getstrm/pace:latest -generatedBufDependencyVersion=00000000000000.64c7ab603c22 +generatedBufDependencyVersion=00000000000000.8c13113d50b0