From ba5d039c2bae781e3271adbcb8d307c07bb59abf Mon Sep 17 00:00:00 2001 From: Mahdi Dibaiee Date: Fri, 15 Mar 2024 15:42:13 +0000 Subject: [PATCH] facebook: ads_insights primary keys must be non-nullable --- .../schemas/activities.json | 3 +- .../schemas/ad_account.json | 3 +- .../schemas/ad_creatives.json | 3 +- .../schemas/ad_sets.json | 3 +- .../schemas/ads.json | 3 +- .../schemas/ads_insights.json | 3 +- .../schemas/ads_insights_breakdowns.json | 16 ++-- .../schemas/campaigns.json | 3 +- .../schemas/images.json | 3 +- .../schemas/videos.json | 3 +- .../streams/base_insight_streams.py | 17 +++- .../snapshots__discover__capture.stdout.json | 86 +++++++++++++++++-- 12 files changed, 123 insertions(+), 23 deletions(-) diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json index 38c9aecded..508d273e5e 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/activities.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/activities.json @@ -37,5 +37,6 @@ "type": ["null", "string"] } }, - "type": ["object"] + "type": ["object"], + "required": ["actor_id", "event_time", "event_type", "object_id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json index cea9a1cdc8..9da6a9b4c7 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_account.json @@ -263,5 +263,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json index 27154319a5..fb944e431e 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json @@ -1375,5 +1375,6 @@ "type": ["null", "string"] } }, - "type": ["object"] + "type": ["object"], + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json index 5abb863f0b..1224293256 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ad_sets.json @@ -110,5 +110,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json index 985e75ab9f..247068d631 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads.json @@ -446,5 +446,6 @@ } } } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json index 600fd6bb3a..42c19161c3 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights.json @@ -338,5 +338,6 @@ "type": ["null", "number"] } }, - "type": ["object"] + "type": ["object"], + "required": ["account_id", "ad_id", "date_start"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json index 5d3227f62a..cb8824fbf5 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/ads_insights_breakdowns.json @@ -1,16 +1,16 @@ { "properties": { "ad_format_asset": { "type": ["null", "string"] }, - "age": { "type": ["string"] }, + "age": { "type": ["null", "string"] }, "app_id": { "type": ["null", "string"] }, "body_asset": { "type": ["null", "string"] }, "call_to_action_asset": { "type": ["null", "string"] }, - "country": { "type": ["string"] }, + "country": { "type": ["null", "string"] }, "description_asset": { "type": ["null", "string"] }, "device_platform": { "type": ["null", "string"] }, - "dma": { "type": ["string"] }, + "dma": { "type": ["null", "string"] }, "frequency_value": { "type": ["null", "string"] }, - "gender": { "type": ["string"] }, + "gender": { "type": ["null", "string"] }, "hourly_stats_aggregated_by_advertiser_time_zone": { "type": ["null", "string"] }, @@ -18,13 +18,13 @@ "type": ["null", "string"] }, "image_asset": { "type": ["null", "string"] }, - "impression_device": { "type": ["string"] }, + "impression_device": { "type": ["null", "string"] }, "link_url_asset": { "type": ["null", "string"] }, "place_page_id": { "type": ["null", "string"] }, - "platform_position": { "type": ["string"] }, + "platform_position": { "type": ["null", "string"] }, "product_id": { "type": ["null", "string"] }, - "publisher_platform": { "type": ["string"] }, - "region": { "type": ["string"] }, + "publisher_platform": { "type": ["null", "string"] }, + "region": { "type": ["null", "string"] }, "skan_conversion_id": { "type": ["null", "string"] }, "title_asset": { "type": ["null", "string"] }, "video_asset": { "type": ["null", "string"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json index 65d27ac88b..a5ec954b7c 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -107,5 +107,6 @@ "type": "string" } }, - "type": ["object"] + "type": ["object"], + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/images.json b/source-facebook-marketing/source_facebook_marketing/schemas/images.json index af05b2296a..1d34c5cdbc 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/images.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/images.json @@ -56,5 +56,6 @@ "width": { "type": ["null", "integer"] } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/schemas/videos.json b/source-facebook-marketing/source_facebook_marketing/schemas/videos.json index 3239f2534b..52e2dac43b 100644 --- a/source-facebook-marketing/source_facebook_marketing/schemas/videos.json +++ b/source-facebook-marketing/source_facebook_marketing/schemas/videos.json @@ -107,5 +107,6 @@ "views": { "type": ["null", "integer"] } - } + }, + "required": ["id"] } diff --git a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 18f180b8cf..6c248a85e6 100644 --- a/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -220,7 +220,6 @@ def check_breakdowns(self): def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - """Slice by date periods and schedule async job for each period, run at most MAX_ASYNC_JOBS jobs at the same time. This solution for Async was chosen because: 1. we should commit state after each successful job @@ -297,6 +296,22 @@ def get_json_schema(self) -> Mapping[str, Any]: if self.breakdowns: breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) + + for k in self.primary_key: + if k not in schema["properties"]: + continue + + tys = schema["properties"][k]["type"] + + if "null" in tys: + schema["properties"][k]["type"].remove("null") + + if "required" not in schema: + schema["required"] = [] + + if k not in schema["required"]: + schema["required"].append(k) + return schema @cached_property diff --git a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json index f1bf9029af..050e256c63 100644 --- a/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json +++ b/source-facebook-marketing/tests/snapshots/snapshots__discover__capture.stdout.json @@ -517,7 +517,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -3192,7 +3195,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -6504,7 +6510,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -12180,6 +12189,9 @@ }, "type": [ "object" + ], + "required": [ + "id" ] }, "key": [ @@ -15906,6 +15918,11 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start" ] }, "key": [ @@ -19644,6 +19661,13 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "age", + "gender" ] }, "key": [ @@ -23379,6 +23403,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "country" ] }, "key": [ @@ -27113,6 +27143,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "region" ] }, "key": [ @@ -30847,6 +30883,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "dma" ] }, "key": [ @@ -34591,6 +34633,14 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "publisher_platform", + "platform_position", + "impression_device" ] }, "key": [ @@ -38322,6 +38372,11 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start" ] }, "key": [ @@ -38517,6 +38572,9 @@ }, "type": [ "object" + ], + "required": [ + "id" ] }, "key": [ @@ -38782,7 +38840,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -38993,7 +39054,10 @@ "row_id" ] } - } + }, + "required": [ + "id" + ] }, "key": [ "/id" @@ -39089,6 +39153,12 @@ }, "type": [ "object" + ], + "required": [ + "actor_id", + "event_time", + "event_type", + "object_id" ] }, "key": [ @@ -39286,6 +39356,12 @@ }, "type": [ "object" + ], + "required": [ + "account_id", + "ad_id", + "date_start", + "publisher_platform" ] }, "key": [