diff --git a/src/dbt/kipptaf/dbt_project.yml b/src/dbt/kipptaf/dbt_project.yml index 34cc0cbcb5..c5e5a61e71 100644 --- a/src/dbt/kipptaf/dbt_project.yml +++ b/src/dbt/kipptaf/dbt_project.yml @@ -58,6 +58,10 @@ models: +schema: google_forms staging: +materialized: table + directory: + +schema: google_directory + staging: + +materialized: table kippadb: +schema: kippadb staging: diff --git a/src/dbt/kipptaf/models/google/directory/sources.yml b/src/dbt/kipptaf/models/google/directory/sources.yml new file mode 100644 index 0000000000..bc2d7d60d1 --- /dev/null +++ b/src/dbt/kipptaf/models/google/directory/sources.yml @@ -0,0 +1,12 @@ +version: 2 + +sources: + - name: google_directory + schema: kipptaf_google_directory + tables: + - name: src_google_directory__users + external: + location: gs://teamster-kipptaf/dagster/kipptaf/google/directory/users/* + options: + format: AVRO + enable_logical_types: true diff --git a/src/dbt/kipptaf/models/google/directory/staging/stg_google_directory__users.sql b/src/dbt/kipptaf/models/google/directory/staging/stg_google_directory__users.sql new file mode 100644 index 0000000000..fb826313fb --- /dev/null +++ b/src/dbt/kipptaf/models/google/directory/staging/stg_google_directory__users.sql @@ -0,0 +1,58 @@ +select + id as `id`, + primaryemail as `primary_email`, + orgunitpath as `org_unit_path`, + recoveryemail as `recovery_email`, + recoveryphone as `recovery_phone`, + suspensionreason as `suspension_reason`, + lastlogintime as `last_login_time`, + creationtime as `creation_time`, + deletiontime as `deletion_time`, + isadmin as `is_admin`, + isdelegatedadmin as `is_delegated_admin`, + isenforcedin2sv as `is_enforced_in_2sv`, + isenrolledin2sv as `is_enrolled_in_2sv`, + ismailboxsetup as `is_mailbox_setup`, + agreedtoterms as `agreed_to_terms`, + archived as `archived`, + suspended as `suspended`, + changepasswordatnextlogin as `change_password_at_next_login`, + includeinglobaladdresslist as `include_in_global_address_list`, + ipwhitelisted as `ip_whitelisted`, + thumbnailphotoetag as `thumbnail_photo_etag`, + thumbnailphotourl as `thumbnail_photo_url`, + hashfunction as `hash_function`, + customerid as `customer_id`, + etag as `etag`, + kind as `kind`, + + {# records #} + gender.addressmeas as `gender__address_me_as`, + gender.customgender as `gender__custom_gender`, + gender.type as `gender__type`, + name.displayname as `name__display_name`, + name.familyname as `name__family_name`, + name.fullname as `name__full_name`, + name.givenname as `name__given_name`, + notes.contenttype as `notes__content_type`, + notes.value as `notes__value`, + + {# repeated #} + aliases as `aliases`, + noneditablealiases as `non_editable_aliases`, + + {# repeated records #} + addresses as `addresses`, + emails as `emails`, + externalids as `external_ids`, + ims as `ims`, + keywords as `keywords`, + languages as `languages`, + locations as `locations`, + organizations as `organizations`, + phones as `phones`, + posixaccounts as `posix_accounts`, + relations as `relations`, + sshpublickeys as `ssh_public_keys`, + websites as `websites`, +from {{ source("google_directory", "src_google_directory__users") }} diff --git a/src/teamster/core/google/directory/assets.py b/src/teamster/core/google/directory/assets.py index 636f72a4e3..a214fedcc6 100644 --- a/src/teamster/core/google/directory/assets.py +++ b/src/teamster/core/google/directory/assets.py @@ -13,7 +13,7 @@ def build_google_directory_assets(code_location): def _asset( context: AssetExecutionContext, google_directory: GoogleDirectoryResource ): - data = google_directory.list_users() + data = google_directory.list_users(projection="full") schema = get_avro_record_schema(name="users", fields=ASSET_FIELDS["users"]) yield Output(value=(data, schema), metadata={"record_count": len(data)}) diff --git a/src/teamster/core/google/directory/resources.py b/src/teamster/core/google/directory/resources.py index dbcc6e8fff..123ea8330c 100644 --- a/src/teamster/core/google/directory/resources.py +++ b/src/teamster/core/google/directory/resources.py @@ -56,3 +56,6 @@ def list_users(self, **kwargs): break return users + + def get_user(self, user_key, **kwargs): + return self._service.users().get(userKey=user_key, **kwargs).execute() diff --git a/src/teamster/core/google/directory/schema.py b/src/teamster/core/google/directory/schema.py index 84bcda3f42..48a2372521 100644 --- a/src/teamster/core/google/directory/schema.py +++ b/src/teamster/core/google/directory/schema.py @@ -125,7 +125,25 @@ {"name": "type", "type": ["null", "string"], "default": None}, ] -CUSTOM_SCHEMA_FIELDS = [] +STUDENT_ATTRIBUTE_FIELDS = [ + {"name": "Student_Number", "type": ["null", "long"], "default": None}, +] + +CUSTOM_SCHEMA_FIELDS = [ + { + "name": "Student_attributes", + "type": [ + "null", + get_avro_record_schema( + name="student_attribute", + fields=STUDENT_ATTRIBUTE_FIELDS, + namespace="user.custom_schema", + ), + ], + "default": None, + }, +] + PASSWORD_FIELDS = [] NAME_FIELDS = [ diff --git a/src/teamster/kipptaf/dbt/assets.py b/src/teamster/kipptaf/dbt/assets.py index dcc90b2b90..dd42ffda15 100644 --- a/src/teamster/kipptaf/dbt/assets.py +++ b/src/teamster/kipptaf/dbt/assets.py @@ -12,7 +12,11 @@ schoolmint, smartrecruiters, ) -from teamster.kipptaf.google.assets import google_forms_assets, google_sheets_assets +from teamster.kipptaf.google.assets import ( + google_directory_assets, + google_forms_assets, + google_sheets_assets, +) dbt_assets = build_dbt_assets(code_location=CODE_LOCATION) @@ -41,12 +45,13 @@ *alchemer.assets, *amplify.assets, *clever.assets, - *google_forms_assets, *iready.assets, *ldap.assets, *renlearn.assets, *schoolmint.assets, *smartrecruiters.assets, + *google_forms_assets, + *google_directory_assets, ] ] diff --git a/src/teamster/kipptaf/google/assets.py b/src/teamster/kipptaf/google/assets.py index 53933fbe0b..6e6bc5994f 100644 --- a/src/teamster/kipptaf/google/assets.py +++ b/src/teamster/kipptaf/google/assets.py @@ -21,10 +21,10 @@ partitions_def=StaticPartitionsDefinition(FORM_IDS), ) -google_directory_assets = build_google_directory_assets(code_location=CODE_LOCATION) +google_directory_assets = [build_google_directory_assets(code_location=CODE_LOCATION)] __all__ = [ *google_sheets_assets, *google_forms_assets, - google_directory_assets, + *google_directory_assets, ] diff --git a/tests/google/directory/test_resources_directory.py b/tests/google/directory/test_resources_directory.py index e16a951b64..bf0ecb60fa 100644 --- a/tests/google/directory/test_resources_directory.py +++ b/tests/google/directory/test_resources_directory.py @@ -4,20 +4,25 @@ from teamster.core.google.directory.resources import GoogleDirectoryResource +with build_resources( + resources={ + "directory": GoogleDirectoryResource( + customer_id="C029u7m0n", + service_account_file_path="/etc/secret-volume/gcloud_service_account_json", + delegated_account="dagster@apps.teamschools.org", + ) + } +) as resources: + DIRECTORY: GoogleDirectoryResource = resources.directory -def test_resource(): - with build_resources( - resources={ - "directory": GoogleDirectoryResource( - customer_id="C029u7m0n", - service_account_file_path="/etc/secret-volume/gcloud_service_account_json", - delegated_account="dagster@apps.teamschools.org", - ) - } - ) as resources: - directory: GoogleDirectoryResource = resources.directory - - users = directory.list_users() + +def test_get_user(): + user = DIRECTORY.get_user(user_key="113203151440162455385") + print(user) + + +def test_list_users(): + users = DIRECTORY.list_users(projection="full") with open(file="env/users.json", mode="w") as f: json.dump(users, f)