update schema

src/dbt/kipptaf/dbt_project.yml

@@ -58,6 +58,10 @@ models:
         +schema: google_forms
         staging:
           +materialized: table
+      directory:
+        +schema: google_directory
+        staging:
+          +materialized: table
     kippadb:
       +schema: kippadb
       staging:

src/dbt/kipptaf/models/google/directory/sources.yml

@@ -0,0 +1,12 @@
+version: 2
+
+sources:
+  - name: google_directory
+    schema: kipptaf_google_directory
+    tables:
+      - name: src_google_directory__users
+        external:
+          location: gs://teamster-kipptaf/dagster/kipptaf/google/directory/users/*
+          options:
+            format: AVRO
+            enable_logical_types: true

src/dbt/kipptaf/models/google/directory/staging/stg_google_directory__users.sql

@@ -0,0 +1,58 @@
+select
+    id as `id`,
+    primaryemail as `primary_email`,
+    orgunitpath as `org_unit_path`,
+    recoveryemail as `recovery_email`,
+    recoveryphone as `recovery_phone`,
+    suspensionreason as `suspension_reason`,
+    lastlogintime as `last_login_time`,
+    creationtime as `creation_time`,
+    deletiontime as `deletion_time`,
+    isadmin as `is_admin`,
+    isdelegatedadmin as `is_delegated_admin`,
+    isenforcedin2sv as `is_enforced_in_2sv`,
+    isenrolledin2sv as `is_enrolled_in_2sv`,
+    ismailboxsetup as `is_mailbox_setup`,
+    agreedtoterms as `agreed_to_terms`,
+    archived as `archived`,
+    suspended as `suspended`,
+    changepasswordatnextlogin as `change_password_at_next_login`,
+    includeinglobaladdresslist as `include_in_global_address_list`,
+    ipwhitelisted as `ip_whitelisted`,
+    thumbnailphotoetag as `thumbnail_photo_etag`,
+    thumbnailphotourl as `thumbnail_photo_url`,
+    hashfunction as `hash_function`,
+    customerid as `customer_id`,
+    etag as `etag`,
+    kind as `kind`,
+
+    {# records #}
+    gender.addressmeas as `gender__address_me_as`,
+    gender.customgender as `gender__custom_gender`,
+    gender.type as `gender__type`,
+    name.displayname as `name__display_name`,
+    name.familyname as `name__family_name`,
+    name.fullname as `name__full_name`,
+    name.givenname as `name__given_name`,
+    notes.contenttype as `notes__content_type`,
+    notes.value as `notes__value`,
+
+    {# repeated #}
+    aliases as `aliases`,
+    noneditablealiases as `non_editable_aliases`,
+
+    {# repeated records #}
+    addresses as `addresses`,
+    emails as `emails`,
+    externalids as `external_ids`,
+    ims as `ims`,
+    keywords as `keywords`,
+    languages as `languages`,
+    locations as `locations`,
+    organizations as `organizations`,
+    phones as `phones`,
+    posixaccounts as `posix_accounts`,
+    relations as `relations`,
+    sshpublickeys as `ssh_public_keys`,
+    websites as `websites`,
+from {{ source("google_directory", "src_google_directory__users") }}

src/teamster/core/google/directory/assets.py

@@ -13,7 +13,7 @@ def build_google_directory_assets(code_location):
     def _asset(
         context: AssetExecutionContext, google_directory: GoogleDirectoryResource
     ):
-        data = google_directory.list_users()
+        data = google_directory.list_users(projection="full")
         schema = get_avro_record_schema(name="users", fields=ASSET_FIELDS["users"])
 
         yield Output(value=(data, schema), metadata={"record_count": len(data)})

src/teamster/core/google/directory/resources.py

@@ -56,3 +56,6 @@ def list_users(self, **kwargs):
                 break
 
         return users
+
+    def get_user(self, user_key, **kwargs):
+        return self._service.users().get(userKey=user_key, **kwargs).execute()

src/teamster/core/google/directory/schema.py

@@ -125,7 +125,25 @@
     {"name": "type", "type": ["null", "string"], "default": None},
 ]
 
-CUSTOM_SCHEMA_FIELDS = []
+STUDENT_ATTRIBUTE_FIELDS = [
+    {"name": "Student_Number", "type": ["null", "long"], "default": None},
+]
+
+CUSTOM_SCHEMA_FIELDS = [
+    {
+        "name": "Student_attributes",
+        "type": [
+            "null",
+            get_avro_record_schema(
+                name="student_attribute",
+                fields=STUDENT_ATTRIBUTE_FIELDS,
+                namespace="user.custom_schema",
+            ),
+        ],
+        "default": None,
+    },
+]
+
 PASSWORD_FIELDS = []
 
 NAME_FIELDS = [

src/teamster/kipptaf/dbt/assets.py

@@ -12,7 +12,11 @@
     schoolmint,
     smartrecruiters,
 )
-from teamster.kipptaf.google.assets import google_forms_assets, google_sheets_assets
+from teamster.kipptaf.google.assets import (
+    google_directory_assets,
+    google_forms_assets,
+    google_sheets_assets,
+)
 
 dbt_assets = build_dbt_assets(code_location=CODE_LOCATION)
 
@@ -41,12 +45,13 @@
         *alchemer.assets,
         *amplify.assets,
         *clever.assets,
-        *google_forms_assets,
         *iready.assets,
         *ldap.assets,
         *renlearn.assets,
         *schoolmint.assets,
         *smartrecruiters.assets,
+        *google_forms_assets,
+        *google_directory_assets,
     ]
 ]
 

src/teamster/kipptaf/google/assets.py

@@ -21,10 +21,10 @@
     partitions_def=StaticPartitionsDefinition(FORM_IDS),
 )
 
-google_directory_assets = build_google_directory_assets(code_location=CODE_LOCATION)
+google_directory_assets = [build_google_directory_assets(code_location=CODE_LOCATION)]
 
 __all__ = [
     *google_sheets_assets,
     *google_forms_assets,
-    google_directory_assets,
+    *google_directory_assets,
 ]

tests/google/directory/test_resources_directory.py

@@ -4,20 +4,25 @@
 
 from teamster.core.google.directory.resources import GoogleDirectoryResource
 
+with build_resources(
+    resources={
+        "directory": GoogleDirectoryResource(
+            customer_id="C029u7m0n",
+            service_account_file_path="/etc/secret-volume/gcloud_service_account_json",
+            delegated_account="dagster@apps.teamschools.org",
+        )
+    }
+) as resources:
+    DIRECTORY: GoogleDirectoryResource = resources.directory
 
-def test_resource():
-    with build_resources(
-        resources={
-            "directory": GoogleDirectoryResource(
-                customer_id="C029u7m0n",
-                service_account_file_path="/etc/secret-volume/gcloud_service_account_json",
-                delegated_account="dagster@apps.teamschools.org",
-            )
-        }
-    ) as resources:
-        directory: GoogleDirectoryResource = resources.directory
-
-    users = directory.list_users()
+
+def test_get_user():
+    user = DIRECTORY.get_user(user_key="113203151440162455385")
+    print(user)
+
+
+def test_list_users():
+    users = DIRECTORY.list_users(projection="full")
 
     with open(file="env/users.json", mode="w") as f:
         json.dump(users, f)