Add Filter and MapToFields transforms for BigQuery to Kafka example

GoogleCloudPlatform · Apr 4, 2024 · 46ff9a2 · 46ff9a2
1 parent 6fccfe6
commit 46ff9a2
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/Python/yaml/bigquery_to_kafka.yaml b/Python/yaml/bigquery_to_kafka.yaml
@@ -20,6 +20,20 @@ pipeline:
     config:
       table: project-id.dataset-id.table-id
 
+  transforms:
+    - type: Filter
+      config:
+        language: python
+        keep: "id > 2"
+    - type: MapToFields
+      config:
+        language: python
+        fields:
+          id:
+            callable: "lambda row: row.id"
+          name:
+            callable: "lambda row: row.name.upper()"
+
   sink:
     type: WriteToKafka
     config: