diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json index 38c9aecded..508d273e5e 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json @@ -37,5 +37,6 @@ "type": ["null", "string"] } }, - "type": ["object"] + "type": ["object"], + "required": ["actor_id", "event_time", "event_type", "object_id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json index cea9a1cdc8..9da6a9b4c7 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json @@ -263,5 +263,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json index 27154319a5..fb944e431e 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json @@ -1375,5 +1375,6 @@ "type": ["null", "string"] } }, - "type": ["object"] + "type": ["object"], + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json index 5abb863f0b..1224293256 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json @@ -110,5 +110,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json index 985e75ab9f..247068d631 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json @@ -446,5 +446,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json index 600fd6bb3a..42c19161c3 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json @@ -338,5 +338,6 @@ "type": ["null", "number"] } }, - "type": ["object"] + "type": ["object"], + "required": ["account_id", "ad_id", "date_start"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json index 5d3227f62a..cb8824fbf5 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json @@ -1,16 +1,16 @@ { "properties": { "ad_format_asset": { "type": ["null", "string"] }, - "age": { "type": ["string"] }, + "age": { "type": ["null", "string"] }, "app_id": { "type": ["null", "string"] }, "body_asset": { "type": ["null", "string"] }, "call_to_action_asset": { "type": ["null", "string"] }, - "country": { "type": ["string"] }, + "country": { "type": ["null", "string"] }, "description_asset": { "type": ["null", "string"] }, "device_platform": { "type": ["null", "string"] }, - "dma": { "type": ["string"] }, + "dma": { "type": ["null", "string"] }, "frequency_value": { "type": ["null", "string"] }, - "gender": { "type": ["string"] }, + "gender": { "type": ["null", "string"] }, "hourly_stats_aggregated_by_advertiser_time_zone": { "type": ["null", "string"] }, @@ -18,13 +18,13 @@ "type": ["null", "string"] }, "image_asset": { "type": ["null", "string"] }, - "impression_device": { "type": ["string"] }, + "impression_device": { "type": ["null", "string"] }, "link_url_asset": { "type": ["null", "string"] }, "place_page_id": { "type": ["null", "string"] }, - "platform_position": { "type": ["string"] }, + "platform_position": { "type": ["null", "string"] }, "product_id": { "type": ["null", "string"] }, - "publisher_platform": { "type": ["string"] }, - "region": { "type": ["string"] }, + "publisher_platform": { "type": ["null", "string"] }, + "region": { "type": ["null", "string"] }, "skan_conversion_id": { "type": ["null", "string"] }, "title_asset": { "type": ["null", "string"] }, "video_asset": { "type": ["null", "string"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json index 65d27ac88b..a5ec954b7c 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -107,5 +107,6 @@ "type": "string" } }, - "type": ["object"] + "type": ["object"], + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/images.json b/source-facebook-marketing/source_facebook_marketing/schemas/images.json index af05b2296a..1d34c5cdbc 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/images.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/images.json @@ -56,5 +56,6 @@ "width": { "type": ["null", "integer"] } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/videos.json b/source-facebook-marketing/source_facebook_marketing/schemas/videos.json index 3239f2534b..52e2dac43b 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/videos.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/videos.json @@ -107,5 +107,6 @@ "views": { "type": ["null", "integer"] } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 18f180b8cf..e8dc2f413c 100644 --- a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -220,7 +220,6 @@ def check_breakdowns(self): def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - """Slice by date periods and schedule async job for each period, run at most MAX_ASYNC_JOBS jobs at the same time. This solution for Async was chosen because: 1. we should commit state after each successful job @@ -297,6 +296,22 @@ def get_json_schema(self) -> Mapping[str, Any]: if self.breakdowns: breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) + + for k in self.primary_key: + if k not in schema["properties"]: + continue + + tys = schema["properties"][k]["type"] + + if "null" in tys: + schema["properties"][k]["type"] = tys.remove("null") + + if "required" not in schema: + schema["required"] = [] + + if k not in schema["required"]: + schema["required"].append(k) + return schema @cached_property diff --git a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json index f1bf9029af..453cda2a54 100644 --- a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json @@ -517,7 +517,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -3192,7 +3195,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -6504,7 +6510,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -12180,6 +12189,9 @@ }, "type": [ "object" + ], + "required": [ + "id" ] }, "key": [ @@ -15906,6 +15918,11 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start" ] }, "key": [ @@ -19621,14 +19638,10 @@ ] }, "age": { - "type": [ - "string" - ] + "type": null }, "gender": { - "type": [ - "string" - ] + "type": null }, "_meta": { "type": "object", @@ -19644,6 +19657,13 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "age", + "gender" ] }, "key": [ @@ -23361,9 +23381,7 @@ ] }, "country": { - "type": [ - "string" - ] + "type": null }, "_meta": { "type": "object", @@ -23379,6 +23397,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "country" ] }, "key": [ @@ -27095,9 +27119,7 @@ ] }, "region": { - "type": [ - "string" - ] + "type": null }, "_meta": { "type": "object", @@ -27113,6 +27135,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "region" ] }, "key": [ @@ -30829,9 +30857,7 @@ ] }, "dma": { - "type": [ - "string" - ] + "type": null }, "_meta": { "type": "object", @@ -30847,6 +30873,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "dma" ] }, "key": [ @@ -34563,19 +34595,13 @@ ] }, "publisher_platform": { - "type": [ - "string" - ] + "type": null }, "platform_position": { - "type": [ - "string" - ] + "type": null }, "impression_device": { - "type": [ - "string" - ] + "type": null }, "_meta": { "type": "object", @@ -34591,6 +34617,14 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "publisher_platform", + "platform_position", + "impression_device" ] }, "key": [ @@ -38322,6 +38356,11 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start" ] }, "key": [ @@ -38517,6 +38556,9 @@ }, "type": [ "object" + ], + "required": [ + "id" ] }, "key": [ @@ -38782,7 +38824,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -38993,7 +39038,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -39089,6 +39137,12 @@ }, "type": [ "object" + ], + "required": [ + "actor_id", + "event_time", + "event_type", + "object_id" ] }, "key": [ @@ -39268,9 +39322,7 @@ ] }, "publisher_platform": { - "type": [ - "string" - ] + "type": null }, "_meta": { "type": "object", @@ -39286,6 +39338,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "publisher_platform" ] }, "key": [