-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
source-google-analytics-data-api: add multiple property ids
- Loading branch information
Showing
7 changed files
with
174 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
85 changes: 85 additions & 0 deletions
85
source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| # | ||
| # Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
| # | ||
|
|
||
|
|
||
| import logging | ||
| from typing import Any, List, Mapping | ||
|
|
||
| from airbyte_cdk.config_observation import create_connector_config_control_message | ||
| from airbyte_cdk.entrypoint import AirbyteEntrypoint | ||
| from airbyte_cdk.sources import Source | ||
| from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository | ||
|
|
||
| logger = logging.getLogger("airbyte_logger") | ||
|
|
||
|
|
||
| class MigrateCustomReports: | ||
| """ | ||
| This class stands for migrating the config at runtime, | ||
| while providing the backward compatibility when falling back to the previous source version. | ||
| Specifically, starting from `1.3.0`, the `property_id` property should be like : | ||
| > List(["<property_id 1>", "<property_id 2>", ..., "<property_id n>"]) | ||
| instead of, in `1.2.0`: | ||
| > JSON STR: "<property_id>" | ||
| """ | ||
|
|
||
| message_repository: MessageRepository = InMemoryMessageRepository() | ||
| migrate_from_key: str = "property_id" | ||
| migrate_to_key: str = "property_ids" | ||
|
|
||
| @classmethod | ||
| def should_migrate(cls, config: Mapping[str, Any]) -> bool: | ||
| """ | ||
| This method determines whether config require migration. | ||
| Returns: | ||
| > True, if the transformation is neccessary | ||
| > False, otherwise. | ||
| """ | ||
| if cls.migrate_from_key in config: | ||
| return True | ||
| return False | ||
|
|
||
| @classmethod | ||
| def transform_to_array(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]: | ||
| # assign old values to new property that will be used within the new version | ||
| config[cls.migrate_to_key] = config[cls.migrate_to_key] if cls.migrate_to_key in config else [] | ||
| data = config.pop(cls.migrate_from_key) | ||
| if data not in config[cls.migrate_to_key]: | ||
| config[cls.migrate_to_key].append(data) | ||
| return config | ||
|
|
||
| @classmethod | ||
| def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: | ||
| # modify the config | ||
| migrated_config = cls.transform_to_array(config, source) | ||
| # save the config | ||
| source.write_config(migrated_config, config_path) | ||
| # return modified config | ||
| return migrated_config | ||
|
|
||
| @classmethod | ||
| def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: | ||
| # add the Airbyte Control Message to message repo | ||
| cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) | ||
| # emit the Airbyte Control Message from message queue to stdout | ||
| for message in cls.message_repository._message_queue: | ||
| print(message.json(exclude_unset=True)) | ||
|
|
||
| @classmethod | ||
| def migrate(cls, args: List[str], source: Source) -> None: | ||
| """ | ||
| This method checks the input args, should the config be migrated, | ||
| transform if neccessary and emit the CONTROL message. | ||
| """ | ||
| # get config path | ||
| config_path = AirbyteEntrypoint(source).extract_config(args) | ||
| # proceed only if `--config` arg is provided | ||
| if config_path: | ||
| # read the existing config | ||
| config = source.read_config(config_path) | ||
| # migration check | ||
| if cls.should_migrate(config): | ||
| cls.emit_control_message( | ||
| cls.modify_and_save(config_path, source, config), | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters