Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bug/renamed columns #46

Closed
wants to merge 7 commits into from
Closed

Bug/renamed columns #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
20658dc
bug/renamed-columns
fivetran-catfritz Jun 29, 2024
9b5db07
add macro file
fivetran-catfritz Jun 29, 2024
fb850fc
add casting
fivetran-catfritz Jun 29, 2024
0a0317b
update yml
fivetran-catfritz Jun 29, 2024
c012fd3
add casting for all
fivetran-catfritz Jul 1, 2024
4a252f2
add casting for all
fivetran-catfritz Jul 1, 2024
f47bd73
macro updates
fivetran-catfritz Jul 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 108 additions & 8 deletions integration_tests/dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,70 +44,133 @@ seeds:
last_activity_date: timestamp
last_referenced_date: timestamp
last_viewed_date: timestamp
account_number: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
annual_revenue: float
billing_latitude: float
billing_longitude: float
parent_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
product_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_city: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_country: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_country_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_postal_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_state: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_state_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_street: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_latitude: float
shipping_longitude: float
sf_opportunity_data:
+column_types:
amount: float
campaign_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
close_date: timestamp
converted_account_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
converted_contact_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
converted_opportunity_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_date: timestamp
individual_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_activity_date: timestamp
last_modified_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_referenced_date: timestamp
last_viewed_date: timestamp
amount: float
master_record_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
mobile_phone: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
next_step: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
owner_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
phone: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
postal_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
probability: float
state_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
synced_quote_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
sf_user_data:
+column_types:
account_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
company_name: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
contact_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
country_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
individual_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_login_date: timestamp
last_referenced_date: timestamp
last_viewed_date: timestamp
offline_trial_expiration_date: timestamp
latitude: float
longitude: float
manager_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
offline_trial_expiration_date: timestamp
postal_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
profile_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
state_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
user_role_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
sf_contact_data:
+column_types:
birthdate: timestamp
created_date: timestamp
email_bounced_date: timestamp
home_phone: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
individual_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
is_deleted: boolean
last_activity_date: timestamp
last_curequest_date: timestamp
last_cuupdate_date: timestamp
last_modified_date: timestamp
last_referenced_date: timestamp
last_viewed_date: timestamp
mailing_postal_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
master_record_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
mobile_phone: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
reports_to_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
system_modstamp: timestamp
is_deleted: boolean
sf_lead_data:
+column_types:
converted_date: timestamp
converted_opportunity_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
country_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_date: timestamp
email_bounced_date: timestamp
email_bounced_reason: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
individual_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_activity_date: timestamp
last_modified_date: timestamp
last_referenced_date: timestamp
last_viewed_date: timestamp
lead_source: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
master_record_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
mobile_phone: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
owner_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
phone: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
postal_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
state_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
system_modstamp: timestamp
is_deleted: boolean
sf_task_data:
+column_types:
account_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
activity_date: timestamp
call_disposition: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
call_object: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
call_type: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
completed_date_time: timestamp
created_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_date: timestamp
is_deleted: boolean
last_modified_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_modified_date: timestamp
owner_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
record_type_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
recurrence_end_date_only: timestamp
recurrence_start_date_only: timestamp
reminder_date_time: timestamp
system_modstamp: timestamp
is_deleted: boolean
what_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
who_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
sf_event_data:
+column_types:
activity_date: timestamp
activity_date_time: timestamp
created_date: timestamp
end_date: timestamp
end_date_time: timestamp
group_event_type: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_modified_date : timestamp
recurrence_2_pattern_start_date : timestamp
recurrence_end_date_only: timestamp
Expand All @@ -118,32 +181,69 @@ seeds:
is_deleted: boolean
sf_product_2_data:
+column_types:
created_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_date: timestamp
display_url: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
external_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
is_deleted: boolean
last_modified_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_modified_date: timestamp
last_referenced_date: timestamp
last_viewed_date: timestamp
product_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
quantity_installment_period: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
quantity_schedule_type: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
quantity_unit_of_measure: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
record_type_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
revenue_installment_period: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
revenue_schedule_type: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
system_modstamp: timestamp
is_deleted: boolean
sf_order_data:
+column_types:
account_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
activated_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
activated_date: timestamp
billing_country_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
billing_state_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
contract_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_date: timestamp
effective_date: timestamp
end_date: timestamp
is_deleted: boolean
last_modified_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_modified_date: timestamp
last_referenced_date: timestamp
last_viewed_date: timestamp
opportunity_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
order_number: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
original_order_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
owner_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
pricebook_2_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_country_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_postal_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
shipping_state_code: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
system_modstamp: timestamp
is_deleted: boolean
sf_opportunity_line_item_data:
+column_types:
campaign_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
created_date: timestamp
has_quantity_schedule: boolean
has_revenue_schedule: boolean
has_schedule: boolean
is_deleted: boolean
last_modified_by_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
last_modified_date: timestamp
last_referenced_date: timestamp
last_viewed_date: timestamp
opportunity_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
product_2_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"
service_date: timestamp
system_modstamp: timestamp
is_deleted: boolean
sf_user_role_data:
+column_types:
parent_role_id: "{{ 'string' if target.name in ['bigquery','spark', 'databricks'] else 'varchar' }}"

sf_account_history_data:
+column_types:
Expand All @@ -154,7 +254,7 @@ seeds:
is_deleted: boolean
sf_contact_history_data:
+column_types:
birthdate: timestamp
birthdate: timestamp
last_activity_date: timestamp
last_modified_date: timestamp
last_viewed_date: timestamp
Expand Down
8 changes: 8 additions & 0 deletions macros/coalesce_w_renamed_col.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{% macro coalesce_w_renamed_col(original_column_name, datatype=dbt.type_string(), alias=original_column_name, renamed_column_name=original_column_name.replace('_', '')) %}
{# This macro accomodates Fivetran connectors that keep the original salesforce field naming conventions without underscores #}

coalesce(cast({{ renamed_column_name }} as {{ datatype }}),
cast({{ original_column_name }} as {{ datatype }}))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know Snowflake provides the ability to enable case-sensitivity (e.g. LastActivityDate). I wonder if this FF does not do the same case-insensitivity approach we see with the standard connector when syncing to Snowflake. I have a small feeling that these original field names may sync like this. Do you know if this could be a problem with the approach we are taking here?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, I'll look into it since right now I'm not sure the best way to handle that scenario. In our example data, everything is lowercased, though I do see in the engineering doc that originally it would be camel case. Let me know if you have any suggestions, but otherwise I'll think on this!

as {{ alias }}

{%- endmacro %}
29 changes: 27 additions & 2 deletions macros/get_account_columns.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,63 @@
{% set columns = [

{"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
{"name": "_fivetran_active", "datatype": "boolean"},
{"name": "_fivetran_active", "datatype": dbt.type_boolean()},
{"name": "account_number", "datatype": dbt.type_string()},
{"name": "accountnumber", "datatype": dbt.type_string()},
{"name": "account_source", "datatype": dbt.type_string()},
{"name": "accountsource", "datatype": dbt.type_string()},
{"name": "annual_revenue", "datatype": dbt.type_float()},
{"name": "annualrevenue", "datatype": dbt.type_float()},
{"name": "billing_city", "datatype": dbt.type_string()},
{"name": "billingcity", "datatype": dbt.type_string()},
{"name": "billing_country", "datatype": dbt.type_string()},
{"name": "billingcountry", "datatype": dbt.type_string()},
{"name": "billing_postal_code", "datatype": dbt.type_string()},
{"name": "billingpostalcode", "datatype": dbt.type_string()},
{"name": "billing_state", "datatype": dbt.type_string()},
{"name": "billingstate", "datatype": dbt.type_string()},
{"name": "billing_state_code", "datatype": dbt.type_string()},
{"name": "billingstatecode", "datatype": dbt.type_string()},
{"name": "billing_street", "datatype": dbt.type_string()},
{"name": "billingstreet", "datatype": dbt.type_string()},
{"name": "description", "datatype": dbt.type_string()},
{"name": "id", "datatype": dbt.type_string()},
{"name": "industry", "datatype": dbt.type_string()},
{"name": "is_deleted", "datatype": "boolean"},
{"name": "is_deleted", "datatype": dbt.type_boolean()},
{"name": "isdeleted", "datatype": dbt.type_boolean()},
{"name": "last_activity_date", "datatype": dbt.type_timestamp()},
{"name": "lastactivitydate", "datatype": dbt.type_timestamp()},
{"name": "last_referenced_date", "datatype": dbt.type_timestamp()},
{"name": "lastreferenceddate", "datatype": dbt.type_timestamp()},
{"name": "last_viewed_date", "datatype": dbt.type_timestamp()},
{"name": "lastvieweddate", "datatype": dbt.type_timestamp()},
{"name": "master_record_id", "datatype": dbt.type_string()},
{"name": "masterrecordid", "datatype": dbt.type_string()},
{"name": "name", "datatype": dbt.type_string()},
{"name": "number_of_employees", "datatype": dbt.type_int()},
{"name": "numberofemployees", "datatype": dbt.type_int()},
{"name": "owner_id", "datatype": dbt.type_string()},
{"name": "ownerid", "datatype": dbt.type_string()},
{"name": "ownership", "datatype": dbt.type_string()},
{"name": "parent_id", "datatype": dbt.type_string()},
{"name": "parentid", "datatype": dbt.type_string()},
{"name": "rating", "datatype": dbt.type_string()},
{"name": "record_type_id", "datatype": dbt.type_string()},
{"name": "recordtypeid", "datatype": dbt.type_string()},
{"name": "shipping_city", "datatype": dbt.type_string()},
{"name": "shippingcity", "datatype": dbt.type_string()},
{"name": "shipping_country", "datatype": dbt.type_string()},
{"name": "shippingcountry", "datatype": dbt.type_string()},
{"name": "shipping_country_code", "datatype": dbt.type_string()},
{"name": "shippingcountrycode", "datatype": dbt.type_string()},
{"name": "shipping_postal_code", "datatype": dbt.type_string()},
{"name": "shippingpostalcode", "datatype": dbt.type_string()},
{"name": "shipping_state", "datatype": dbt.type_string()},
{"name": "shippingstate", "datatype": dbt.type_string()},
{"name": "shipping_state_code", "datatype": dbt.type_string()},
{"name": "shippingstatecode", "datatype": dbt.type_string()},
{"name": "shipping_street", "datatype": dbt.type_string()},
{"name": "shippingstreet", "datatype": dbt.type_string()},
{"name": "type", "datatype": dbt.type_string()},
{"name": "website", "datatype": dbt.type_string()}
] %}
Expand Down
Loading