Merge pull request #1377 from estuary/mahdi/fb-keys

facebook: ads_insights primary keys must be non-nullable
estuary · Mar 15, 2024 · a9ec02d · a9ec02d
2 parents d59e97e + ba5d039
commit a9ec02d
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 23 deletions.
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json
@@ -37,5 +37,6 @@
       "type": ["null", "string"]
     }
   },
-  "type": ["object"]
+  "type": ["object"],
+  "required": ["actor_id", "event_time", "event_type", "object_id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json
@@ -263,5 +263,6 @@
         }
       }
     }
-  }
+  },
+  "required": ["id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json
@@ -1375,5 +1375,6 @@
       "type": ["null", "string"]
     }
   },
-  "type": ["object"]
+  "type": ["object"],
+  "required": ["id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json
@@ -110,5 +110,6 @@
         }
       }
     }
-  }
+  },
+  "required": ["id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json
@@ -446,5 +446,6 @@
         }
       }
     }
-  }
+  },
+  "required": ["id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json
@@ -338,5 +338,6 @@
       "type": ["null", "number"]
     }
   },
-  "type": ["object"]
+  "type": ["object"],
+  "required": ["account_id", "ad_id", "date_start"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json
@@ -1,30 +1,30 @@
 {
   "properties": {
     "ad_format_asset": { "type": ["null", "string"] },
-    "age": { "type": ["string"] },
+    "age": { "type": ["null", "string"] },
     "app_id": { "type": ["null", "string"] },
     "body_asset": { "type": ["null", "string"] },
     "call_to_action_asset": { "type": ["null", "string"] },
-    "country": { "type": ["string"] },
+    "country": { "type": ["null", "string"] },
     "description_asset": { "type": ["null", "string"] },
     "device_platform": { "type": ["null", "string"] },
-    "dma": { "type": ["string"] },
+    "dma": { "type": ["null", "string"] },
     "frequency_value": { "type": ["null", "string"] },
-    "gender": { "type": ["string"] },
+    "gender": { "type": ["null", "string"] },
     "hourly_stats_aggregated_by_advertiser_time_zone": {
       "type": ["null", "string"]
     },
     "hourly_stats_aggregated_by_audience_time_zone": {
       "type": ["null", "string"]
     },
     "image_asset": { "type": ["null", "string"] },
-    "impression_device": { "type": ["string"] },
+    "impression_device": { "type": ["null", "string"] },
     "link_url_asset": { "type": ["null", "string"] },
     "place_page_id": { "type": ["null", "string"] },
-    "platform_position": { "type": ["string"] },
+    "platform_position": { "type": ["null", "string"] },
     "product_id": { "type": ["null", "string"] },
-    "publisher_platform": { "type": ["string"] },
-    "region": { "type": ["string"] },
+    "publisher_platform": { "type": ["null", "string"] },
+    "region": { "type": ["null", "string"] },
     "skan_conversion_id": { "type": ["null", "string"] },
     "title_asset": { "type": ["null", "string"] },
     "video_asset": { "type": ["null", "string"] }

diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json
@@ -107,5 +107,6 @@
       "type": "string"
     }
   },
-  "type": ["object"]
+  "type": ["object"],
+  "required": ["id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/images.json b/source-facebook-marketing/source_facebook_marketing/schemas/images.json
@@ -56,5 +56,6 @@
     "width": {
       "type": ["null", "integer"]
     }
-  }
+  },
+  "required": ["id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/videos.json b/source-facebook-marketing/source_facebook_marketing/schemas/videos.json
@@ -107,5 +107,6 @@
     "views": {
       "type": ["null", "integer"]
     }
-  }
+  },
+  "required": ["id"]
 }
diff --git a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py
@@ -220,7 +220,6 @@ def check_breakdowns(self):
     def stream_slices(
         self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
     ) -> Iterable[Optional[Mapping[str, Any]]]:
-
         """Slice by date periods and schedule async job for each period, run at most MAX_ASYNC_JOBS jobs at the same time.
         This solution for Async was chosen because:
         1. we should commit state after each successful job
@@ -297,6 +296,22 @@ def get_json_schema(self) -> Mapping[str, Any]:
         if self.breakdowns:
             breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"]
             schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns})
+
+        for k in self.primary_key:
+            if k not in schema["properties"]:
+                continue
+
+            tys = schema["properties"][k]["type"]
+
+            if "null" in tys:
+                schema["properties"][k]["type"].remove("null")
+
+            if "required" not in schema:
+                schema["required"] = []
+
+            if k not in schema["required"]:
+                schema["required"].append(k)
+
         return schema
 
     @cached_property

diff --git a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json
@@ -517,7 +517,10 @@
             "row_id"
           ]
         }
-      }
+      },
+      "required": [
+        "id"
+      ]
     },
     "key": [
       "/id"
@@ -3192,7 +3195,10 @@
             "row_id"
           ]
         }
-      }
+      },
+      "required": [
+        "id"
+      ]
     },
     "key": [
       "/id"
@@ -6504,7 +6510,10 @@
             "row_id"
           ]
         }
-      }
+      },
+      "required": [
+        "id"
+      ]
     },
     "key": [
       "/id"
@@ -12180,6 +12189,9 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "id"
       ]
     },
     "key": [
@@ -15906,6 +15918,11 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start"
       ]
     },
     "key": [
@@ -19644,6 +19661,13 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start",
+        "age",
+        "gender"
       ]
     },
     "key": [
@@ -23379,6 +23403,12 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start",
+        "country"
       ]
     },
     "key": [
@@ -27113,6 +27143,12 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start",
+        "region"
       ]
     },
     "key": [
@@ -30847,6 +30883,12 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start",
+        "dma"
       ]
     },
     "key": [
@@ -34591,6 +34633,14 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start",
+        "publisher_platform",
+        "platform_position",
+        "impression_device"
       ]
     },
     "key": [
@@ -38322,6 +38372,11 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start"
       ]
     },
     "key": [
@@ -38517,6 +38572,9 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "id"
       ]
     },
     "key": [
@@ -38782,7 +38840,10 @@
             "row_id"
           ]
         }
-      }
+      },
+      "required": [
+        "id"
+      ]
     },
     "key": [
       "/id"
@@ -38993,7 +39054,10 @@
             "row_id"
           ]
         }
-      }
+      },
+      "required": [
+        "id"
+      ]
     },
     "key": [
       "/id"
@@ -39089,6 +39153,12 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "actor_id",
+        "event_time",
+        "event_type",
+        "object_id"
       ]
     },
     "key": [
@@ -39286,6 +39356,12 @@
       },
       "type": [
         "object"
+      ],
+      "required": [
+        "account_id",
+        "ad_id",
+        "date_start",
+        "publisher_platform"
       ]
     },
     "key": [
-Original file line number
+Diff line change
@@ Expand Up / @@ -263,5 +263,6 @@ @@
             }
           }
         }
-      }
+      },
+      "required": ["id"]
     }