diff --git a/source-impact-native/source_impact_native/__init__.py b/source-impact-native/source_impact_native/__init__.py index 134a46ad8b..5a6db3749c 100644 --- a/source-impact-native/source_impact_native/__init__.py +++ b/source-impact-native/source_impact_native/__init__.py @@ -29,7 +29,7 @@ def request_class(self): async def spec(self, log: Logger, _: request.Spec) -> ConnectorSpec: return ConnectorSpec( - documentationUrl="https://docs.estuary.dev", + documentationUrl="https://go.estuary.dev/source-impact-native", configSchema=EndpointConfig.model_json_schema(), oauth2=None, resourceConfigSchema=ResourceConfig.model_json_schema(), diff --git a/source-impact-native/source_impact_native/api.py b/source-impact-native/source_impact_native/api.py index 08d4378024..5f43244914 100644 --- a/source-impact-native/source_impact_native/api.py +++ b/source-impact-native/source_impact_native/api.py @@ -58,7 +58,7 @@ async def fetch_incremental( async def fetch_backfill( cls, - stop_date, + config_start_date: datetime, account_sid, http: HTTPSession, log: Logger, @@ -80,16 +80,16 @@ async def fetch_backfill( else: url = f"{API}/{API_CATALOG}/{account_sid}/{cls.NAME}" if _cls.START_DATE: - parameters = {f"{_cls.START_DATE}": _cursor_dt(cls.NAME, stop_date), f"{_cls.END_DATE}": _cursor_dt(cls.NAME, cutoff)} + parameters = {f"{_cls.START_DATE}": _cursor_dt(cls.NAME, config_start_date), f"{_cls.END_DATE}": _cursor_dt(cls.NAME, cutoff)} result = json.loads(await http.request(log, url, method="GET", params=parameters, headers=headers)) for results in result[f"{_cls.NAME}"]: - if _s_to_dt(results[f"{_cls.REP_KEY}"]) == stop_date: + if _s_to_dt(results[f"{_cls.REP_KEY}"]) == config_start_date: doc = _cls.model_validate_json(json.dumps(results)) yield doc return - elif _s_to_dt(results[f"{_cls.REP_KEY}"]) < stop_date: + elif _s_to_dt(results[f"{_cls.REP_KEY}"]) < config_start_date: return elif _s_to_dt(results[f"{_cls.REP_KEY}"]) < cutoff: doc = _cls.model_validate_json(json.dumps(results)) @@ -156,7 +156,7 @@ async def fetch_incremental_actions( async def fetch_backfill_actions( cls_parent, cls, - stop_date, + config_start_date: datetime, account_sid, http: HTTPSession, log: Logger, @@ -176,8 +176,8 @@ async def fetch_backfill_actions( parameters = {} - if cutoff - timedelta(days=1095) <= stop_date <= cutoff: - start_date = stop_date + if cutoff - timedelta(days=1095) <= config_start_date <= cutoff: + start_date = config_start_date end_date = cutoff else: start_date = cutoff - timedelta(days=1095) @@ -202,8 +202,8 @@ async def fetch_backfill_actions( for start, end in dates: iterating = True - parameters["LockingDateStart"] = _cursor_dt(cls.NAME, start) - parameters["LockingDateEnd"] = _cursor_dt(cls.NAME, end) + parameters["ActionDateStart"] = _cursor_dt(cls.NAME, start) + parameters["ActionDateEnd"] = _cursor_dt(cls.NAME, end) while iterating: @@ -211,12 +211,12 @@ async def fetch_backfill_actions( result = json.loads(await http.request(log, url, method="GET", params=parameters, headers=headers)) for results in result[f"{_cls.NAME}"]: - if _s_to_dt(results[f"CreationDate"]) == stop_date: + if _s_to_dt(results[f"CreationDate"]) == config_start_date: doc = _cls.model_validate_json(json.dumps(results)) yield doc iterating = False break - elif _s_to_dt(results[f"CreationDate"]) < stop_date: + elif _s_to_dt(results[f"CreationDate"]) < config_start_date: break elif _s_to_dt(results[f"CreationDate"]) < cutoff: doc = _cls.model_validate_json(json.dumps(results)) @@ -316,7 +316,7 @@ async def fetch_incremental_child( async def fetch_backfill_child( cls_parent, cls, - stop_date, + config_start_date: datetime, account_sid, http: HTTPSession, log: Logger, @@ -349,17 +349,17 @@ async def fetch_backfill_child( url = f"{API}/{API_CATALOG}/{account_sid}/Programs/{campaign}/{cls.NAME}" headers = {'Accept': 'application/json'} # if _cls.START_DATE: - # parameters = {f"{_cls.START_DATE}": _cursor_dt(cls.NAME, stop_date), f"{_cls.END_DATE}": _cursor_dt(cls.NAME, cutoff)} + # parameters = {f"{_cls.START_DATE}": _cursor_dt(cls.NAME, config_start_date), f"{_cls.END_DATE}": _cursor_dt(cls.NAME, cutoff)} while iterating: result = json.loads(await http.request(log, url, method="GET", params=parameters, headers=headers)) for results in result[f"{_cls.NAME}"]: - if _s_to_dt(results[f"{_cls.REP_KEY}"]) == stop_date: + if _s_to_dt(results[f"{_cls.REP_KEY}"]) == config_start_date: doc = _cls.model_validate_json(json.dumps(results)) yield doc - elif _s_to_dt(results[f"{_cls.REP_KEY}"]) < stop_date: + elif _s_to_dt(results[f"{_cls.REP_KEY}"]) < config_start_date: iterating = False break elif _s_to_dt(results[f"{_cls.REP_KEY}"]) < cutoff: diff --git a/source-impact-native/source_impact_native/models.py b/source-impact-native/source_impact_native/models.py index 3d4474c652..b77b2e0cec 100644 --- a/source-impact-native/source_impact_native/models.py +++ b/source-impact-native/source_impact_native/models.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, UTC, timedelta from enum import StrEnum, auto from pydantic import BaseModel, Field, AwareDatetime from typing import Literal, Generic, TypeVar, Annotated, ClassVar, TYPE_CHECKING, Dict, List @@ -17,6 +17,10 @@ ConnectorState = GenericConnectorState[ResourceState] +def default_start_date(): + dt = datetime.now(tz=UTC) - timedelta(days=30) + return dt + class CatalogEnum(StrEnum): brand = "Brand" @@ -31,10 +35,10 @@ class EndpointConfig(BaseModel, extra="allow"): description="As of now, only BRAND catalogs are allowed" ) - stop_date: AwareDatetime = Field( - description="Replication Stop Date. Records will only be considered for backfilling "\ - "before the stop_date, similar to a start date", - default=datetime.fromisoformat("2010-01-01T00:00:00Z".replace('Z', '+00:00')) + start_date: AwareDatetime = Field( + description="UTC date and time in the format YYYY-MM-DDTHH:MM:SSZ. Data generated before this date will not be replicated. If left blank, the start date will be set to 30 days before the present date.", + title="Start Date", + default_factory=default_start_date ) diff --git a/source-impact-native/source_impact_native/resources.py b/source-impact-native/source_impact_native/resources.py index b5a8664f24..1d982514f5 100644 --- a/source-impact-native/source_impact_native/resources.py +++ b/source-impact-native/source_impact_native/resources.py @@ -50,25 +50,25 @@ async def all_resources( if config.api_catalog == "Brand": # We pass along the credentials.username value since its required by every endpoint all_streams = ( - action_object(Campaigns, Actions, http, config.stop_date, config.credentials.username), - action_object(Campaigns, ActionInquiries, http, config.stop_date, config.credentials.username), - base_object(Invoices, http, config.stop_date, config.credentials.username), - base_object(Jobs, http, config.stop_date, config.credentials.username), - base_object(PromoCodes, http, config.stop_date, config.credentials.username), - base_object(Deals, http, config.stop_date, config.credentials.username), - base_object(ExceptionLists, http, config.stop_date, config.credentials.username), - base_object(UniqueUrls, http, config.stop_date, config.credentials.username), - base_object(TrackingValueRequests, http, config.stop_date, config.credentials.username), - snapshot_object(Campaigns, http, config.stop_date, config.credentials.username), - snapshot_object(Ads, http, config.stop_date, config.credentials.username), - snapshot_object(Catalogs, http, config.stop_date, config.credentials.username), - snapshot_object(Reports, http, config.stop_date, config.credentials.username), - snapshot_object(PhoneNumbers, http, config.stop_date, config.credentials.username), - child_object(Campaigns, Contracts, http, config.stop_date, config.credentials.username), - child_object(Campaigns, Tasks, http, config.stop_date, config.credentials.username), - child_object(Campaigns, Notes, http, config.stop_date, config.credentials.username), - child_object(Campaigns, BlockRedirectRules, http, config.stop_date, config.credentials.username), - media_groups_object(Campaigns, MediaPartnerGroups, http, config.stop_date, config.credentials.username), + action_object(Campaigns, Actions, http, config.start_date, config.credentials.username), + action_object(Campaigns, ActionInquiries, http, config.start_date, config.credentials.username), + base_object(Invoices, http, config.start_date, config.credentials.username), + base_object(Jobs, http, config.start_date, config.credentials.username), + base_object(PromoCodes, http, config.start_date, config.credentials.username), + base_object(Deals, http, config.start_date, config.credentials.username), + base_object(ExceptionLists, http, config.start_date, config.credentials.username), + base_object(UniqueUrls, http, config.start_date, config.credentials.username), + base_object(TrackingValueRequests, http, config.start_date, config.credentials.username), + snapshot_object(Campaigns, http, config.start_date, config.credentials.username), + snapshot_object(Ads, http, config.start_date, config.credentials.username), + snapshot_object(Catalogs, http, config.start_date, config.credentials.username), + snapshot_object(Reports, http, config.start_date, config.credentials.username), + snapshot_object(PhoneNumbers, http, config.start_date, config.credentials.username), + child_object(Campaigns, Contracts, http, config.start_date, config.credentials.username), + child_object(Campaigns, Tasks, http, config.start_date, config.credentials.username), + child_object(Campaigns, Notes, http, config.start_date, config.credentials.username), + child_object(Campaigns, BlockRedirectRules, http, config.start_date, config.credentials.username), + media_groups_object(Campaigns, MediaPartnerGroups, http, config.start_date, config.credentials.username), ) @@ -114,7 +114,7 @@ def open( inc=ResourceState.Incremental(cursor=started_at), backfill=ResourceState.Backfill(next_page=None, cutoff=started_at) ), - initial_config=ResourceConfig(name=cls.NAME), + initial_config=ResourceConfig(name=cls.NAME, interval=timedelta(minutes=5)), schema_inference=True, ) @@ -154,7 +154,7 @@ def open( inc=ResourceState.Incremental(cursor=started_at), backfill=ResourceState.Backfill(next_page=None, cutoff=started_at) ), - initial_config=ResourceConfig(name=cls.NAME), + initial_config=ResourceConfig(name=cls.NAME, interval=timedelta(minutes=5)), schema_inference=True, ) @@ -187,7 +187,7 @@ def open( model=cls, open=open, initial_state=ResourceState(), - initial_config=ResourceConfig(name=cls.NAME), + initial_config=ResourceConfig(name=cls.NAME, interval=timedelta(minutes=5)), schema_inference=True, ) @@ -225,7 +225,7 @@ def open( inc=ResourceState.Incremental(cursor=started_at), backfill=ResourceState.Backfill(next_page=None, cutoff=started_at) ), - initial_config=ResourceConfig(name=cls.NAME), + initial_config=ResourceConfig(name=cls.NAME, interval=timedelta(minutes=5)), schema_inference=True, ) @@ -259,6 +259,6 @@ def open( model=cls, open=open, initial_state=ResourceState(), - initial_config=ResourceConfig(name=cls.NAME), + initial_config=ResourceConfig(name=cls.NAME, interval=timedelta(minutes=5)), schema_inference=True, ) \ No newline at end of file diff --git a/source-impact-native/tests/__init__.py b/source-impact-native/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__discover__stdout.json b/source-impact-native/tests/snapshots/snapshots__discover__stdout.json similarity index 96% rename from source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__discover__stdout.json rename to source-impact-native/tests/snapshots/snapshots__discover__stdout.json index 51fdcab8df..ed106d5afb 100644 --- a/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__discover__stdout.json +++ b/source-impact-native/tests/snapshots/snapshots__discover__stdout.json @@ -2,7 +2,8 @@ { "recommendedName": "Actions", "resourceConfig": { - "name": "Actions" + "name": "Actions", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -63,7 +64,8 @@ { "recommendedName": "ActionInquiries", "resourceConfig": { - "name": "ActionInquiries" + "name": "ActionInquiries", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -124,7 +126,8 @@ { "recommendedName": "Invoices", "resourceConfig": { - "name": "Invoices" + "name": "Invoices", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -185,7 +188,8 @@ { "recommendedName": "Jobs", "resourceConfig": { - "name": "Jobs" + "name": "Jobs", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -246,7 +250,8 @@ { "recommendedName": "PromoCodes", "resourceConfig": { - "name": "PromoCodes" + "name": "PromoCodes", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -307,7 +312,8 @@ { "recommendedName": "Deals", "resourceConfig": { - "name": "Deals" + "name": "Deals", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -368,7 +374,8 @@ { "recommendedName": "ExceptionLists", "resourceConfig": { - "name": "ExceptionLists" + "name": "ExceptionLists", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -429,7 +436,8 @@ { "recommendedName": "UniqueUrls", "resourceConfig": { - "name": "UniqueUrls" + "name": "UniqueUrls", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -490,7 +498,8 @@ { "recommendedName": "TrackingValueRequests", "resourceConfig": { - "name": "TrackingValueRequests" + "name": "TrackingValueRequests", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -551,7 +560,8 @@ { "recommendedName": "Campaigns", "resourceConfig": { - "name": "Campaigns" + "name": "Campaigns", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -605,7 +615,8 @@ { "recommendedName": "Ads", "resourceConfig": { - "name": "Ads" + "name": "Ads", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -659,7 +670,8 @@ { "recommendedName": "Catalogs", "resourceConfig": { - "name": "Catalogs" + "name": "Catalogs", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -713,7 +725,8 @@ { "recommendedName": "Reports", "resourceConfig": { - "name": "Reports" + "name": "Reports", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -767,7 +780,8 @@ { "recommendedName": "PhoneNumbers", "resourceConfig": { - "name": "PhoneNumbers" + "name": "PhoneNumbers", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -821,7 +835,8 @@ { "recommendedName": "Contracts", "resourceConfig": { - "name": "Contracts" + "name": "Contracts", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -882,7 +897,8 @@ { "recommendedName": "Tasks", "resourceConfig": { - "name": "Tasks" + "name": "Tasks", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -943,7 +959,8 @@ { "recommendedName": "Notes", "resourceConfig": { - "name": "Notes" + "name": "Notes", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -1004,7 +1021,8 @@ { "recommendedName": "BlockRedirectRules", "resourceConfig": { - "name": "BlockRedirectRules" + "name": "BlockRedirectRules", + "interval": "PT5M" }, "documentSchema": { "$defs": { @@ -1065,7 +1083,8 @@ { "recommendedName": "MediaPartnerGroups", "resourceConfig": { - "name": "MediaPartnerGroups" + "name": "MediaPartnerGroups", + "interval": "PT5M" }, "documentSchema": { "$defs": { diff --git a/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__spec__stdout.json b/source-impact-native/tests/snapshots/snapshots__spec__stdout.json similarity index 87% rename from source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__spec__stdout.json rename to source-impact-native/tests/snapshots/snapshots__spec__stdout.json index 5f4127ba59..ea8ea8722e 100644 --- a/source-impact-native/tests/snapshots/source_impact_native_tests_test_snapshots__spec__stdout.json +++ b/source-impact-native/tests/snapshots/snapshots__spec__stdout.json @@ -60,11 +60,10 @@ "description": "As of now, only BRAND catalogs are allowed", "title": "API Catalog (Brand, Agency, Partner)" }, - "stop_date": { - "default": "2010-01-01T00:00:00Z", - "description": "Replication Stop Date. Records will only be considered for backfilling before the stop_date, similar to a start date", + "start_date": { + "description": "UTC date and time in the format YYYY-MM-DDTHH:MM:SSZ. Data generated before this date will not be replicated. If left blank, the start date will be set to 30 days before the present date.", "format": "date-time", - "title": "Stop Date", + "title": "Start Date", "type": "string" } }, @@ -98,7 +97,7 @@ "title": "ResourceConfig", "type": "object" }, - "documentationUrl": "https://docs.estuary.dev", + "documentationUrl": "https://go.estuary.dev/source-impact-native", "resourcePathPointers": [ "/name" ]