From 3586551102a3b6a2d39fc50cfac8382f404c9e81 Mon Sep 17 00:00:00 2001 From: wesleycheung0 Date: Wed, 15 May 2024 00:23:13 -0400 Subject: [PATCH 1/5] Update the script to take spreadsheet data dictionary as input --- .../scripts/generate_syh_dr_data_models.py | 206 ++++++++---------- 1 file changed, 93 insertions(+), 113 deletions(-) diff --git a/data_processing/scripts/generate_syh_dr_data_models.py b/data_processing/scripts/generate_syh_dr_data_models.py index eacef23..5ad2a65 100644 --- a/data_processing/scripts/generate_syh_dr_data_models.py +++ b/data_processing/scripts/generate_syh_dr_data_models.py @@ -2,122 +2,110 @@ import sys import pandas as pd import requests -import tempfile -import pdfplumber - - -def process_csv_files(pdf_url, csv_folder): - # Download the data dictionary PDF - response = requests.get(pdf_url) - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - temp_file.write(response.content) - pdf_path = temp_file.name - - # Load the PDF file - pdf = pdfplumber.open(pdf_path) +from bs4 import BeautifulSoup +import re +import ssl +from pathlib import Path + +def create_data_dictionary(): + """ + 1. Set CONSTANTS for project page + 2. Get links to data dictionary csv files + 3. Create list of data dictionary urls + 4. Concat all data dictionary files into a dataframe + @chrisgebert + """ + AHRQ_BASE_URL = 'https://www.ahrq.gov' + SYH_DR_URL = 'https://www.ahrq.gov/data/innovations/syh-dr.html' + + # Get all links to Data Dictionary CSV files + response = requests.get(SYH_DR_URL) + soup = BeautifulSoup(response.text, 'html.parser') + + # Create list of data dictionary URLs (i.e., where attribute is like "Variables") + data_dictionary_urls = [] + for url in soup.find_all("a", string=re.compile("Variables")): + csv_url = AHRQ_BASE_URL + url.attrs.get("href") + data_dictionary_urls.append(csv_url) + + # Concat all data dictionaries into a dataframe and write to a file + ssl._create_default_https_context = ssl._create_unverified_context + data_dictionary_df = pd.concat((pd.read_csv(url) for url in data_dictionary_urls), ignore_index=True) + data_dictionary_df = data_dictionary_df[data_dictionary_df["Variable Name"].notnull()] + data_dictionary_df.to_csv("./data_processing/seeds/SyH-DR_data_dictionary.csv", index=False) + + return data_dictionary_df[["Variable Label", "Variable Name", "Variable Type", "Variable Format", "Variable Length"]] + +def map_variable_type_to_sql(variable_name, variable_type, variable_length, variable_format): + if "_ID" in variable_name: + return "UBIGINT" + elif "LOS" in variable_name: + return "UINTEGER" + elif "_DT" in variable_name or "_DATE" in variable_name: + return "DATE" + elif "DOLLAR" in variable_format: + return "FLOAT" + elif variable_type == "NUM": + return f"NUMERIC" + elif variable_type == "CHAR": + return f"VARCHAR" + else: + return "TEXT" + +def process_csv_files(data_dictionary_df, csv_folder): + csv_folder = Path(csv_folder) # Define the CSV file names - csv_files = [ - "syhdr_commercial_inpatient_2016.CSV", - "syhdr_commercial_outpatient_2016.CSV", - "syhdr_commercial_person_2016.CSV", - "syhdr_commercial_pharmacy_2016.CSV", - "syhdr_medicaid_inpatient_2016.CSV", - "syhdr_medicaid_outpatient_2016.CSV", - "syhdr_medicaid_person_2016.CSV", - "syhdr_medicaid_pharmacy_2016.CSV", - "syhdr_medicaid_provider_2016.csv", - "syhdr_medicare_inpatient_2016.CSV", - "syhdr_medicare_outpatient_2016.CSV", - "syhdr_medicare_person_2016.CSV", - "syhdr_medicare_pharmacy_2016.CSV", - "syhdr_medicare_provider_2016.csv", - ] + csv_files = [f for f in csv_folder.glob('*.CSV')] # Process each CSV file for csv_file in csv_files: - csv_path = os.path.join(csv_folder, csv_file) - if not os.path.exists(csv_path): - print(f"File not found: {csv_path}") + if not csv_file.exists(): + print(f"File not found: {csv_file}") continue - # Load the CSV file into a pandas DataFrame - df = pd.read_csv(csv_path, nrows=1) + try: + df = pd.read_csv(csv_file, nrows=1) + except Exception as e: + print(f"Error reading CSV file {csv_file}: {e}") + continue - # Get the column names from the DataFrame column_names = df.columns.tolist() - - print(f"Processing file: {csv_file}") - print("Column names:") - print(column_names) - print() - - # Initialize a dictionary to store the data types for each column column_definitions = {} - - # Search for each column name in the PDF pages - csv_types = None + for column_name in column_names: - print(f"Processing column: {column_name}") - data_type = None - # Manually set the data type for some columns - if "_DT" in column_name or "_DATE" in column_name: - data_type = "DATE" - elif "_ST_CD" in column_name: - data_type = "VARCHAR" - elif "CPT" in column_name: - data_type = "VARCHAR" - if csv_types is None: - csv_types = f"'{column_name}': 'VARCHAR'" - else: - csv_types += f", '{column_name}': 'VARCHAR'" - elif "_ID" in column_name and "CD" not in column_name: - data_type = "UBIGINT" - elif "LOS" in column_name: - data_type = "UINTEGER" - elif "AMT" in column_name: - data_type = "FLOAT" - - for page_number in range(10, len(pdf.pages)): - print(f"Searching for column '{column_name}' on page {page_number + 1}") - page = pdf.pages[page_number] - cropped_page = page.crop( - (72, 86.4, page.width - 72, page.height - 70.0) + match = data_dictionary_df[data_dictionary_df["Variable Name"] == column_name] + if not match.empty: + variable_type = match["Variable Type"].iloc[0] + variable_length = match["Variable Length"].iloc[0] + variable_format = match["Variable Format"].iloc[0] if "Variable Format" in match.columns else "" + column_definitions[column_name] = map_variable_type_to_sql( + column_name, variable_type, variable_length, variable_format ) - text = cropped_page.extract_text() - lines = text.split("\n") + else: + # Treat as VARCHAR if not found in the data dictionary + column_definitions[column_name] = 'VARCHAR' - if column_name in text: - print(f"Column '{column_name}' found on page {page_number + 1}") - - # Extract the first occurrence of "Character" or "Numeric" before the table start - if data_type is None: - if "Character" in lines: - data_type = "VARCHAR" - elif "Numeric" in lines: - data_type = "NUMERIC" - print(f"Data type for column '{column_name}': {data_type}") - - if data_type: - column_definitions[column_name] = data_type - break # Break the loop once the column is found and data type is extracted - - print() + print(f"Processing file: {csv_file}") + print("Column names and their assigned types from data dictionary:") # Generate the SQL model - sql_file = os.path.splitext(csv_file)[0] + ".sql" - with open(os.path.join("models/ahrq.gov/generated/with_types", sql_file), "w") as f: + sql_file = os.path.splitext(csv_file.name)[0] + ".sql" + sql_file_path = os.path.join("data_processing/models/ahrq.gov/generated/with_types", sql_file) + with open(sql_file_path, "w") as f: # Write the SQL model header - f.write(f"-- SQL model for {csv_file}\n") + f.write(f"-- SQL model for {csv_file.name}\n") f.write( "{{ config(materialized='external', location=var('output_path') + '/' + this.name + '.parquet') }}\n\n" ) # Write the SQL SELECT statement column_list = [] + types_list = [] for column_name in column_names: if column_name in column_definitions: data_type = column_definitions[column_name] + types_list.append(f"'{column_name}': '{data_type}'") if "AMT" in column_name: # remove dollar sign from the VARCHAR string column_list.append( @@ -129,34 +117,26 @@ def process_csv_files(pdf_url, csv_folder): ) else: column_list.append(f"{column_name}::VARCHAR") + types_list.append(f"'{column_name}': 'VARCHAR'") - if csv_types: - open_bracket = "{" - close_bracket = "}" - csv_str = f", types={open_bracket}{csv_types}{close_bracket}, ignore_errors=true" - print(csv_str) + types_str = ", ".join(types_list) username = os.environ.get("USER") - path_without_user = "~/" + csv_path.split(username + '/')[1] - select_statement = f"SELECT\n {',\n '.join(column_list)}\nFROM read_csv('{path_without_user}', header=True, null_padding=true{csv_str if csv_types else ''})" + path_without_user = "~/" + str(csv_file).split(username + '/')[1] + select_statement = ( + "SELECT\n " + ",\n ".join(column_list) + "\n" + + f"FROM read_csv('{path_without_user}', header=True, null_padding=true, types={{ {types_str} }}, ignore_errors=true)" + ) f.write(select_statement) - print(f"Generated SQL model: {sql_file}") - print() - - # Clean up the temporary PDF file - os.unlink(pdf_path) - - -def main(csv_folder, data_dictionary_url): - process_csv_files(data_dictionary_url, csv_folder) + print(f"Generated SQL model: {sql_file_path}") +def main(csv_folder): + data_dictionary_df = create_data_dictionary() + process_csv_files(data_dictionary_df, csv_folder) if __name__ == "__main__": - if len(sys.argv) != 3: - print( - "Usage: python generate_syh_dr_sql_models.py " - ) + if len(sys.argv) != 2: + print("Usage: python generate_syh_dr_data_models.py ") sys.exit(1) csv_folder = sys.argv[1] - data_dictionary_url = sys.argv[2] - main(csv_folder, data_dictionary_url) + main(csv_folder) From 614df031b975735887cb02e0404c1a0c7bffecda Mon Sep 17 00:00:00 2001 From: wesleycheung0 Date: Wed, 15 May 2024 00:23:46 -0400 Subject: [PATCH 2/5] Updated data model --- .../syhdr_commercial_inpatient_2016.sql | 156 +++++++++--------- .../syhdr_commercial_outpatient_2016.sql | 156 +++++++++--------- .../syhdr_commercial_person_2016.sql | 44 ++--- .../syhdr_commercial_pharmacy_2016.sql | 6 +- .../syhdr_medicaid_inpatient_2016.sql | 156 +++++++++--------- .../syhdr_medicaid_outpatient_2016.sql | 156 +++++++++--------- .../with_types/syhdr_medicaid_person_2016.sql | 66 ++++---- .../syhdr_medicaid_pharmacy_2016.sql | 6 +- .../syhdr_medicaid_provider_2016.sql | 12 +- .../syhdr_medicare_inpatient_2016.sql | 156 +++++++++--------- .../syhdr_medicare_outpatient_2016.sql | 156 +++++++++--------- .../with_types/syhdr_medicare_person_2016.sql | 84 +++++----- .../syhdr_medicare_pharmacy_2016.sql | 6 +- .../syhdr_medicare_provider_2016.sql | 12 +- 14 files changed, 586 insertions(+), 586 deletions(-) diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql index 88fe090..1c82356 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql @@ -14,93 +14,93 @@ SELECT TOB_CD::VARCHAR AS TOB_CD, CLM_TYPE_CD::VARCHAR AS CLM_TYPE_CD, DSCHRG_STUS::VARCHAR AS DSCHRG_STUS, - PRMRY_DX_IMPUTED::VARCHAR AS PRMRY_DX_IMPUTED, + PRMRY_DX_IMPUTED::NUMERIC AS PRMRY_DX_IMPUTED, PRMRY_DX_CD::VARCHAR AS PRMRY_DX_CD, ICD_DX_CD_1::VARCHAR AS ICD_DX_CD_1, ICD_DX_CD_2::VARCHAR AS ICD_DX_CD_2, - ICD_DX_CD_3::VARCHAR, - ICD_DX_CD_4::VARCHAR, - ICD_DX_CD_5::VARCHAR, - ICD_DX_CD_6::VARCHAR, - ICD_DX_CD_7::VARCHAR, - ICD_DX_CD_8::VARCHAR, - ICD_DX_CD_9::VARCHAR, - ICD_DX_CD_10::VARCHAR, - ICD_DX_CD_11::VARCHAR, - ICD_DX_CD_12::VARCHAR, - ICD_DX_CD_13::VARCHAR, - ICD_DX_CD_14::VARCHAR, - ICD_DX_CD_15::VARCHAR, - ICD_DX_CD_16::VARCHAR, - ICD_DX_CD_17::VARCHAR, - ICD_DX_CD_18::VARCHAR, - ICD_DX_CD_19::VARCHAR, - ICD_DX_CD_20::VARCHAR, - ICD_DX_CD_21::VARCHAR, - ICD_DX_CD_22::VARCHAR, - ICD_DX_CD_23::VARCHAR, - ICD_DX_CD_24::VARCHAR, + ICD_DX_CD_3::VARCHAR AS ICD_DX_CD_3, + ICD_DX_CD_4::VARCHAR AS ICD_DX_CD_4, + ICD_DX_CD_5::VARCHAR AS ICD_DX_CD_5, + ICD_DX_CD_6::VARCHAR AS ICD_DX_CD_6, + ICD_DX_CD_7::VARCHAR AS ICD_DX_CD_7, + ICD_DX_CD_8::VARCHAR AS ICD_DX_CD_8, + ICD_DX_CD_9::VARCHAR AS ICD_DX_CD_9, + ICD_DX_CD_10::VARCHAR AS ICD_DX_CD_10, + ICD_DX_CD_11::VARCHAR AS ICD_DX_CD_11, + ICD_DX_CD_12::VARCHAR AS ICD_DX_CD_12, + ICD_DX_CD_13::VARCHAR AS ICD_DX_CD_13, + ICD_DX_CD_14::VARCHAR AS ICD_DX_CD_14, + ICD_DX_CD_15::VARCHAR AS ICD_DX_CD_15, + ICD_DX_CD_16::VARCHAR AS ICD_DX_CD_16, + ICD_DX_CD_17::VARCHAR AS ICD_DX_CD_17, + ICD_DX_CD_18::VARCHAR AS ICD_DX_CD_18, + ICD_DX_CD_19::VARCHAR AS ICD_DX_CD_19, + ICD_DX_CD_20::VARCHAR AS ICD_DX_CD_20, + ICD_DX_CD_21::VARCHAR AS ICD_DX_CD_21, + ICD_DX_CD_22::VARCHAR AS ICD_DX_CD_22, + ICD_DX_CD_23::VARCHAR AS ICD_DX_CD_23, + ICD_DX_CD_24::VARCHAR AS ICD_DX_CD_24, ICD_DX_CD_25::VARCHAR AS ICD_DX_CD_25, ICD_PRCDR_CD_1::VARCHAR AS ICD_PRCDR_CD_1, ICD_PRCDR_CD_2::VARCHAR AS ICD_PRCDR_CD_2, - ICD_PRCDR_CD_3::VARCHAR, - ICD_PRCDR_CD_4::VARCHAR, - ICD_PRCDR_CD_5::VARCHAR, - ICD_PRCDR_CD_6::VARCHAR, - ICD_PRCDR_CD_7::VARCHAR, - ICD_PRCDR_CD_8::VARCHAR, - ICD_PRCDR_CD_9::VARCHAR, - ICD_PRCDR_CD_10::VARCHAR, - ICD_PRCDR_CD_11::VARCHAR, - ICD_PRCDR_CD_12::VARCHAR, - ICD_PRCDR_CD_13::VARCHAR, - ICD_PRCDR_CD_14::VARCHAR, - ICD_PRCDR_CD_15::VARCHAR, - ICD_PRCDR_CD_16::VARCHAR, - ICD_PRCDR_CD_17::VARCHAR, - ICD_PRCDR_CD_18::VARCHAR, - ICD_PRCDR_CD_19::VARCHAR, - ICD_PRCDR_CD_20::VARCHAR, - ICD_PRCDR_CD_21::VARCHAR, - ICD_PRCDR_CD_22::VARCHAR, - ICD_PRCDR_CD_23::VARCHAR, - ICD_PRCDR_CD_24::VARCHAR, + ICD_PRCDR_CD_3::VARCHAR AS ICD_PRCDR_CD_3, + ICD_PRCDR_CD_4::VARCHAR AS ICD_PRCDR_CD_4, + ICD_PRCDR_CD_5::VARCHAR AS ICD_PRCDR_CD_5, + ICD_PRCDR_CD_6::VARCHAR AS ICD_PRCDR_CD_6, + ICD_PRCDR_CD_7::VARCHAR AS ICD_PRCDR_CD_7, + ICD_PRCDR_CD_8::VARCHAR AS ICD_PRCDR_CD_8, + ICD_PRCDR_CD_9::VARCHAR AS ICD_PRCDR_CD_9, + ICD_PRCDR_CD_10::VARCHAR AS ICD_PRCDR_CD_10, + ICD_PRCDR_CD_11::VARCHAR AS ICD_PRCDR_CD_11, + ICD_PRCDR_CD_12::VARCHAR AS ICD_PRCDR_CD_12, + ICD_PRCDR_CD_13::VARCHAR AS ICD_PRCDR_CD_13, + ICD_PRCDR_CD_14::VARCHAR AS ICD_PRCDR_CD_14, + ICD_PRCDR_CD_15::VARCHAR AS ICD_PRCDR_CD_15, + ICD_PRCDR_CD_16::VARCHAR AS ICD_PRCDR_CD_16, + ICD_PRCDR_CD_17::VARCHAR AS ICD_PRCDR_CD_17, + ICD_PRCDR_CD_18::VARCHAR AS ICD_PRCDR_CD_18, + ICD_PRCDR_CD_19::VARCHAR AS ICD_PRCDR_CD_19, + ICD_PRCDR_CD_20::VARCHAR AS ICD_PRCDR_CD_20, + ICD_PRCDR_CD_21::VARCHAR AS ICD_PRCDR_CD_21, + ICD_PRCDR_CD_22::VARCHAR AS ICD_PRCDR_CD_22, + ICD_PRCDR_CD_23::VARCHAR AS ICD_PRCDR_CD_23, + ICD_PRCDR_CD_24::VARCHAR AS ICD_PRCDR_CD_24, ICD_PRCDR_CD_25::VARCHAR AS ICD_PRCDR_CD_25, CPT_PRCDR_CD_1::VARCHAR AS CPT_PRCDR_CD_1, - CPT_PRCDR_CD_2::VARCHAR, + CPT_PRCDR_CD_2::VARCHAR AS CPT_PRCDR_CD_2, CPT_PRCDR_CD_3::VARCHAR AS CPT_PRCDR_CD_3, - CPT_PRCDR_CD_4::VARCHAR, - CPT_PRCDR_CD_5::VARCHAR, - CPT_PRCDR_CD_6::VARCHAR, - CPT_PRCDR_CD_7::VARCHAR, - CPT_PRCDR_CD_8::VARCHAR, - CPT_PRCDR_CD_9::VARCHAR, - CPT_PRCDR_CD_10::VARCHAR, - CPT_PRCDR_CD_11::VARCHAR, - CPT_PRCDR_CD_12::VARCHAR, - CPT_PRCDR_CD_13::VARCHAR, - CPT_PRCDR_CD_14::VARCHAR, - CPT_PRCDR_CD_15::VARCHAR, - CPT_PRCDR_CD_16::VARCHAR, - CPT_PRCDR_CD_17::VARCHAR, - CPT_PRCDR_CD_18::VARCHAR, - CPT_PRCDR_CD_19::VARCHAR, - CPT_PRCDR_CD_20::VARCHAR, - CPT_PRCDR_CD_21::VARCHAR, - CPT_PRCDR_CD_22::VARCHAR, - CPT_PRCDR_CD_23::VARCHAR, - CPT_PRCDR_CD_24::VARCHAR, - CPT_PRCDR_CD_25::VARCHAR, - CPT_PRCDR_CD_26::VARCHAR, - CPT_PRCDR_CD_27::VARCHAR, - CPT_PRCDR_CD_28::VARCHAR, - CPT_PRCDR_CD_29::VARCHAR, - CPT_PRCDR_CD_30::VARCHAR, - CPT_PRCDR_CD_31::VARCHAR, - CPT_PRCDR_CD_32::VARCHAR, - CPT_PRCDR_CD_33::VARCHAR, - CPT_PRCDR_CD_34::VARCHAR, + CPT_PRCDR_CD_4::VARCHAR AS CPT_PRCDR_CD_4, + CPT_PRCDR_CD_5::VARCHAR AS CPT_PRCDR_CD_5, + CPT_PRCDR_CD_6::VARCHAR AS CPT_PRCDR_CD_6, + CPT_PRCDR_CD_7::VARCHAR AS CPT_PRCDR_CD_7, + CPT_PRCDR_CD_8::VARCHAR AS CPT_PRCDR_CD_8, + CPT_PRCDR_CD_9::VARCHAR AS CPT_PRCDR_CD_9, + CPT_PRCDR_CD_10::VARCHAR AS CPT_PRCDR_CD_10, + CPT_PRCDR_CD_11::VARCHAR AS CPT_PRCDR_CD_11, + CPT_PRCDR_CD_12::VARCHAR AS CPT_PRCDR_CD_12, + CPT_PRCDR_CD_13::VARCHAR AS CPT_PRCDR_CD_13, + CPT_PRCDR_CD_14::VARCHAR AS CPT_PRCDR_CD_14, + CPT_PRCDR_CD_15::VARCHAR AS CPT_PRCDR_CD_15, + CPT_PRCDR_CD_16::VARCHAR AS CPT_PRCDR_CD_16, + CPT_PRCDR_CD_17::VARCHAR AS CPT_PRCDR_CD_17, + CPT_PRCDR_CD_18::VARCHAR AS CPT_PRCDR_CD_18, + CPT_PRCDR_CD_19::VARCHAR AS CPT_PRCDR_CD_19, + CPT_PRCDR_CD_20::VARCHAR AS CPT_PRCDR_CD_20, + CPT_PRCDR_CD_21::VARCHAR AS CPT_PRCDR_CD_21, + CPT_PRCDR_CD_22::VARCHAR AS CPT_PRCDR_CD_22, + CPT_PRCDR_CD_23::VARCHAR AS CPT_PRCDR_CD_23, + CPT_PRCDR_CD_24::VARCHAR AS CPT_PRCDR_CD_24, + CPT_PRCDR_CD_25::VARCHAR AS CPT_PRCDR_CD_25, + CPT_PRCDR_CD_26::VARCHAR AS CPT_PRCDR_CD_26, + CPT_PRCDR_CD_27::VARCHAR AS CPT_PRCDR_CD_27, + CPT_PRCDR_CD_28::VARCHAR AS CPT_PRCDR_CD_28, + CPT_PRCDR_CD_29::VARCHAR AS CPT_PRCDR_CD_29, + CPT_PRCDR_CD_30::VARCHAR AS CPT_PRCDR_CD_30, + CPT_PRCDR_CD_31::VARCHAR AS CPT_PRCDR_CD_31, + CPT_PRCDR_CD_32::VARCHAR AS CPT_PRCDR_CD_32, + CPT_PRCDR_CD_33::VARCHAR AS CPT_PRCDR_CD_33, + CPT_PRCDR_CD_34::VARCHAR AS CPT_PRCDR_CD_34, CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_commercial_inpatient_2016.CSV', header=True, null_padding=true, types={'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR'}, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_commercial_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql index 67daf0a..fa79128 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql @@ -14,93 +14,93 @@ SELECT TOB_CD::VARCHAR AS TOB_CD, CLM_TYPE_CD::VARCHAR AS CLM_TYPE_CD, DSCHRG_STUS::VARCHAR AS DSCHRG_STUS, - PRMRY_DX_IMPUTED::VARCHAR AS PRMRY_DX_IMPUTED, + PRMRY_DX_IMPUTED::NUMERIC AS PRMRY_DX_IMPUTED, PRMRY_DX_CD::VARCHAR AS PRMRY_DX_CD, ICD_DX_CD_1::VARCHAR AS ICD_DX_CD_1, ICD_DX_CD_2::VARCHAR AS ICD_DX_CD_2, - ICD_DX_CD_3::VARCHAR, - ICD_DX_CD_4::VARCHAR, - ICD_DX_CD_5::VARCHAR, - ICD_DX_CD_6::VARCHAR, - ICD_DX_CD_7::VARCHAR, - ICD_DX_CD_8::VARCHAR, - ICD_DX_CD_9::VARCHAR, - ICD_DX_CD_10::VARCHAR, - ICD_DX_CD_11::VARCHAR, - ICD_DX_CD_12::VARCHAR, - ICD_DX_CD_13::VARCHAR, - ICD_DX_CD_14::VARCHAR, - ICD_DX_CD_15::VARCHAR, - ICD_DX_CD_16::VARCHAR, - ICD_DX_CD_17::VARCHAR, - ICD_DX_CD_18::VARCHAR, - ICD_DX_CD_19::VARCHAR, - ICD_DX_CD_20::VARCHAR, - ICD_DX_CD_21::VARCHAR, - ICD_DX_CD_22::VARCHAR, - ICD_DX_CD_23::VARCHAR, - ICD_DX_CD_24::VARCHAR, + ICD_DX_CD_3::VARCHAR AS ICD_DX_CD_3, + ICD_DX_CD_4::VARCHAR AS ICD_DX_CD_4, + ICD_DX_CD_5::VARCHAR AS ICD_DX_CD_5, + ICD_DX_CD_6::VARCHAR AS ICD_DX_CD_6, + ICD_DX_CD_7::VARCHAR AS ICD_DX_CD_7, + ICD_DX_CD_8::VARCHAR AS ICD_DX_CD_8, + ICD_DX_CD_9::VARCHAR AS ICD_DX_CD_9, + ICD_DX_CD_10::VARCHAR AS ICD_DX_CD_10, + ICD_DX_CD_11::VARCHAR AS ICD_DX_CD_11, + ICD_DX_CD_12::VARCHAR AS ICD_DX_CD_12, + ICD_DX_CD_13::VARCHAR AS ICD_DX_CD_13, + ICD_DX_CD_14::VARCHAR AS ICD_DX_CD_14, + ICD_DX_CD_15::VARCHAR AS ICD_DX_CD_15, + ICD_DX_CD_16::VARCHAR AS ICD_DX_CD_16, + ICD_DX_CD_17::VARCHAR AS ICD_DX_CD_17, + ICD_DX_CD_18::VARCHAR AS ICD_DX_CD_18, + ICD_DX_CD_19::VARCHAR AS ICD_DX_CD_19, + ICD_DX_CD_20::VARCHAR AS ICD_DX_CD_20, + ICD_DX_CD_21::VARCHAR AS ICD_DX_CD_21, + ICD_DX_CD_22::VARCHAR AS ICD_DX_CD_22, + ICD_DX_CD_23::VARCHAR AS ICD_DX_CD_23, + ICD_DX_CD_24::VARCHAR AS ICD_DX_CD_24, ICD_DX_CD_25::VARCHAR AS ICD_DX_CD_25, ICD_PRCDR_CD_1::VARCHAR AS ICD_PRCDR_CD_1, ICD_PRCDR_CD_2::VARCHAR AS ICD_PRCDR_CD_2, - ICD_PRCDR_CD_3::VARCHAR, - ICD_PRCDR_CD_4::VARCHAR, - ICD_PRCDR_CD_5::VARCHAR, - ICD_PRCDR_CD_6::VARCHAR, - ICD_PRCDR_CD_7::VARCHAR, - ICD_PRCDR_CD_8::VARCHAR, - ICD_PRCDR_CD_9::VARCHAR, - ICD_PRCDR_CD_10::VARCHAR, - ICD_PRCDR_CD_11::VARCHAR, - ICD_PRCDR_CD_12::VARCHAR, - ICD_PRCDR_CD_13::VARCHAR, - ICD_PRCDR_CD_14::VARCHAR, - ICD_PRCDR_CD_15::VARCHAR, - ICD_PRCDR_CD_16::VARCHAR, - ICD_PRCDR_CD_17::VARCHAR, - ICD_PRCDR_CD_18::VARCHAR, - ICD_PRCDR_CD_19::VARCHAR, - ICD_PRCDR_CD_20::VARCHAR, - ICD_PRCDR_CD_21::VARCHAR, - ICD_PRCDR_CD_22::VARCHAR, - ICD_PRCDR_CD_23::VARCHAR, - ICD_PRCDR_CD_24::VARCHAR, + ICD_PRCDR_CD_3::VARCHAR AS ICD_PRCDR_CD_3, + ICD_PRCDR_CD_4::VARCHAR AS ICD_PRCDR_CD_4, + ICD_PRCDR_CD_5::VARCHAR AS ICD_PRCDR_CD_5, + ICD_PRCDR_CD_6::VARCHAR AS ICD_PRCDR_CD_6, + ICD_PRCDR_CD_7::VARCHAR AS ICD_PRCDR_CD_7, + ICD_PRCDR_CD_8::VARCHAR AS ICD_PRCDR_CD_8, + ICD_PRCDR_CD_9::VARCHAR AS ICD_PRCDR_CD_9, + ICD_PRCDR_CD_10::VARCHAR AS ICD_PRCDR_CD_10, + ICD_PRCDR_CD_11::VARCHAR AS ICD_PRCDR_CD_11, + ICD_PRCDR_CD_12::VARCHAR AS ICD_PRCDR_CD_12, + ICD_PRCDR_CD_13::VARCHAR AS ICD_PRCDR_CD_13, + ICD_PRCDR_CD_14::VARCHAR AS ICD_PRCDR_CD_14, + ICD_PRCDR_CD_15::VARCHAR AS ICD_PRCDR_CD_15, + ICD_PRCDR_CD_16::VARCHAR AS ICD_PRCDR_CD_16, + ICD_PRCDR_CD_17::VARCHAR AS ICD_PRCDR_CD_17, + ICD_PRCDR_CD_18::VARCHAR AS ICD_PRCDR_CD_18, + ICD_PRCDR_CD_19::VARCHAR AS ICD_PRCDR_CD_19, + ICD_PRCDR_CD_20::VARCHAR AS ICD_PRCDR_CD_20, + ICD_PRCDR_CD_21::VARCHAR AS ICD_PRCDR_CD_21, + ICD_PRCDR_CD_22::VARCHAR AS ICD_PRCDR_CD_22, + ICD_PRCDR_CD_23::VARCHAR AS ICD_PRCDR_CD_23, + ICD_PRCDR_CD_24::VARCHAR AS ICD_PRCDR_CD_24, ICD_PRCDR_CD_25::VARCHAR AS ICD_PRCDR_CD_25, CPT_PRCDR_CD_1::VARCHAR AS CPT_PRCDR_CD_1, - CPT_PRCDR_CD_2::VARCHAR, + CPT_PRCDR_CD_2::VARCHAR AS CPT_PRCDR_CD_2, CPT_PRCDR_CD_3::VARCHAR AS CPT_PRCDR_CD_3, - CPT_PRCDR_CD_4::VARCHAR, - CPT_PRCDR_CD_5::VARCHAR, - CPT_PRCDR_CD_6::VARCHAR, - CPT_PRCDR_CD_7::VARCHAR, - CPT_PRCDR_CD_8::VARCHAR, - CPT_PRCDR_CD_9::VARCHAR, - CPT_PRCDR_CD_10::VARCHAR, - CPT_PRCDR_CD_11::VARCHAR, - CPT_PRCDR_CD_12::VARCHAR, - CPT_PRCDR_CD_13::VARCHAR, - CPT_PRCDR_CD_14::VARCHAR, - CPT_PRCDR_CD_15::VARCHAR, - CPT_PRCDR_CD_16::VARCHAR, - CPT_PRCDR_CD_17::VARCHAR, - CPT_PRCDR_CD_18::VARCHAR, - CPT_PRCDR_CD_19::VARCHAR, - CPT_PRCDR_CD_20::VARCHAR, - CPT_PRCDR_CD_21::VARCHAR, - CPT_PRCDR_CD_22::VARCHAR, - CPT_PRCDR_CD_23::VARCHAR, - CPT_PRCDR_CD_24::VARCHAR, - CPT_PRCDR_CD_25::VARCHAR, - CPT_PRCDR_CD_26::VARCHAR, - CPT_PRCDR_CD_27::VARCHAR, - CPT_PRCDR_CD_28::VARCHAR, - CPT_PRCDR_CD_29::VARCHAR, - CPT_PRCDR_CD_30::VARCHAR, - CPT_PRCDR_CD_31::VARCHAR, - CPT_PRCDR_CD_32::VARCHAR, - CPT_PRCDR_CD_33::VARCHAR, - CPT_PRCDR_CD_34::VARCHAR, + CPT_PRCDR_CD_4::VARCHAR AS CPT_PRCDR_CD_4, + CPT_PRCDR_CD_5::VARCHAR AS CPT_PRCDR_CD_5, + CPT_PRCDR_CD_6::VARCHAR AS CPT_PRCDR_CD_6, + CPT_PRCDR_CD_7::VARCHAR AS CPT_PRCDR_CD_7, + CPT_PRCDR_CD_8::VARCHAR AS CPT_PRCDR_CD_8, + CPT_PRCDR_CD_9::VARCHAR AS CPT_PRCDR_CD_9, + CPT_PRCDR_CD_10::VARCHAR AS CPT_PRCDR_CD_10, + CPT_PRCDR_CD_11::VARCHAR AS CPT_PRCDR_CD_11, + CPT_PRCDR_CD_12::VARCHAR AS CPT_PRCDR_CD_12, + CPT_PRCDR_CD_13::VARCHAR AS CPT_PRCDR_CD_13, + CPT_PRCDR_CD_14::VARCHAR AS CPT_PRCDR_CD_14, + CPT_PRCDR_CD_15::VARCHAR AS CPT_PRCDR_CD_15, + CPT_PRCDR_CD_16::VARCHAR AS CPT_PRCDR_CD_16, + CPT_PRCDR_CD_17::VARCHAR AS CPT_PRCDR_CD_17, + CPT_PRCDR_CD_18::VARCHAR AS CPT_PRCDR_CD_18, + CPT_PRCDR_CD_19::VARCHAR AS CPT_PRCDR_CD_19, + CPT_PRCDR_CD_20::VARCHAR AS CPT_PRCDR_CD_20, + CPT_PRCDR_CD_21::VARCHAR AS CPT_PRCDR_CD_21, + CPT_PRCDR_CD_22::VARCHAR AS CPT_PRCDR_CD_22, + CPT_PRCDR_CD_23::VARCHAR AS CPT_PRCDR_CD_23, + CPT_PRCDR_CD_24::VARCHAR AS CPT_PRCDR_CD_24, + CPT_PRCDR_CD_25::VARCHAR AS CPT_PRCDR_CD_25, + CPT_PRCDR_CD_26::VARCHAR AS CPT_PRCDR_CD_26, + CPT_PRCDR_CD_27::VARCHAR AS CPT_PRCDR_CD_27, + CPT_PRCDR_CD_28::VARCHAR AS CPT_PRCDR_CD_28, + CPT_PRCDR_CD_29::VARCHAR AS CPT_PRCDR_CD_29, + CPT_PRCDR_CD_30::VARCHAR AS CPT_PRCDR_CD_30, + CPT_PRCDR_CD_31::VARCHAR AS CPT_PRCDR_CD_31, + CPT_PRCDR_CD_32::VARCHAR AS CPT_PRCDR_CD_32, + CPT_PRCDR_CD_33::VARCHAR AS CPT_PRCDR_CD_33, + CPT_PRCDR_CD_34::VARCHAR AS CPT_PRCDR_CD_34, CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_commercial_outpatient_2016.CSV', header=True, null_padding=true, types={'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR'}, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_commercial_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_person_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_person_2016.sql index 04ed28e..e42c12c 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_person_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_person_2016.sql @@ -6,32 +6,32 @@ SELECT PERSON_WGHT::NUMERIC AS PERSON_WGHT, AGE_LOW::NUMERIC AS AGE_LOW, AGE_HIGH::NUMERIC AS AGE_HIGH, - SEX_IDENT_CD::VARCHAR AS SEX_IDENT_CD, + SEX_IDENT_CD::UBIGINT AS SEX_IDENT_CD, STATE_CD::VARCHAR AS STATE_CD, COUNTY_FIPS_CD::VARCHAR AS COUNTY_FIPS_CD, ZIP_CD::VARCHAR AS ZIP_CD, PHRMCY_CVRG_1::NUMERIC AS PHRMCY_CVRG_1, - PHRMCY_CVRG_2::VARCHAR, - PHRMCY_CVRG_3::VARCHAR, - PHRMCY_CVRG_4::VARCHAR, - PHRMCY_CVRG_5::VARCHAR, - PHRMCY_CVRG_6::VARCHAR, - PHRMCY_CVRG_7::VARCHAR, - PHRMCY_CVRG_8::VARCHAR, - PHRMCY_CVRG_9::VARCHAR, - PHRMCY_CVRG_10::VARCHAR, - PHRMCY_CVRG_11::VARCHAR, + PHRMCY_CVRG_2::NUMERIC AS PHRMCY_CVRG_2, + PHRMCY_CVRG_3::NUMERIC AS PHRMCY_CVRG_3, + PHRMCY_CVRG_4::NUMERIC AS PHRMCY_CVRG_4, + PHRMCY_CVRG_5::NUMERIC AS PHRMCY_CVRG_5, + PHRMCY_CVRG_6::NUMERIC AS PHRMCY_CVRG_6, + PHRMCY_CVRG_7::NUMERIC AS PHRMCY_CVRG_7, + PHRMCY_CVRG_8::NUMERIC AS PHRMCY_CVRG_8, + PHRMCY_CVRG_9::NUMERIC AS PHRMCY_CVRG_9, + PHRMCY_CVRG_10::NUMERIC AS PHRMCY_CVRG_10, + PHRMCY_CVRG_11::NUMERIC AS PHRMCY_CVRG_11, PHRMCY_CVRG_12::NUMERIC AS PHRMCY_CVRG_12, CMRCL_INSRC_1::NUMERIC AS CMRCL_INSRC_1, - CMRCL_INSRC_2::VARCHAR, - CMRCL_INSRC_3::VARCHAR, - CMRCL_INSRC_4::VARCHAR, - CMRCL_INSRC_5::VARCHAR, - CMRCL_INSRC_6::VARCHAR, - CMRCL_INSRC_7::VARCHAR, - CMRCL_INSRC_8::VARCHAR, - CMRCL_INSRC_9::VARCHAR, - CMRCL_INSRC_10::VARCHAR, - CMRCL_INSRC_11::VARCHAR, + CMRCL_INSRC_2::NUMERIC AS CMRCL_INSRC_2, + CMRCL_INSRC_3::NUMERIC AS CMRCL_INSRC_3, + CMRCL_INSRC_4::NUMERIC AS CMRCL_INSRC_4, + CMRCL_INSRC_5::NUMERIC AS CMRCL_INSRC_5, + CMRCL_INSRC_6::NUMERIC AS CMRCL_INSRC_6, + CMRCL_INSRC_7::NUMERIC AS CMRCL_INSRC_7, + CMRCL_INSRC_8::NUMERIC AS CMRCL_INSRC_8, + CMRCL_INSRC_9::NUMERIC AS CMRCL_INSRC_9, + CMRCL_INSRC_10::NUMERIC AS CMRCL_INSRC_10, + CMRCL_INSRC_11::NUMERIC AS CMRCL_INSRC_11, CMRCL_INSRC_12::NUMERIC AS CMRCL_INSRC_12 -FROM read_csv('/Users/me/data/syh_dr/syhdr_commercial_person_2016.CSV', header=True, null_padding=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_commercial_person_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'AGE_LOW': 'NUMERIC', 'AGE_HIGH': 'NUMERIC', 'SEX_IDENT_CD': 'UBIGINT', 'STATE_CD': 'VARCHAR', 'COUNTY_FIPS_CD': 'VARCHAR', 'ZIP_CD': 'VARCHAR', 'PHRMCY_CVRG_1': 'NUMERIC', 'PHRMCY_CVRG_2': 'NUMERIC', 'PHRMCY_CVRG_3': 'NUMERIC', 'PHRMCY_CVRG_4': 'NUMERIC', 'PHRMCY_CVRG_5': 'NUMERIC', 'PHRMCY_CVRG_6': 'NUMERIC', 'PHRMCY_CVRG_7': 'NUMERIC', 'PHRMCY_CVRG_8': 'NUMERIC', 'PHRMCY_CVRG_9': 'NUMERIC', 'PHRMCY_CVRG_10': 'NUMERIC', 'PHRMCY_CVRG_11': 'NUMERIC', 'PHRMCY_CVRG_12': 'NUMERIC', 'CMRCL_INSRC_1': 'NUMERIC', 'CMRCL_INSRC_2': 'NUMERIC', 'CMRCL_INSRC_3': 'NUMERIC', 'CMRCL_INSRC_4': 'NUMERIC', 'CMRCL_INSRC_5': 'NUMERIC', 'CMRCL_INSRC_6': 'NUMERIC', 'CMRCL_INSRC_7': 'NUMERIC', 'CMRCL_INSRC_8': 'NUMERIC', 'CMRCL_INSRC_9': 'NUMERIC', 'CMRCL_INSRC_10': 'NUMERIC', 'CMRCL_INSRC_11': 'NUMERIC', 'CMRCL_INSRC_12': 'NUMERIC' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql index 2f78560..e706ff0 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql @@ -6,10 +6,10 @@ SELECT PERSON_WGHT::NUMERIC AS PERSON_WGHT, PHMCY_CLM_NUM::NUMERIC AS PHMCY_CLM_NUM, CLM_CNTL_NUM::NUMERIC AS CLM_CNTL_NUM, - LINE_NBR::VARCHAR, + LINE_NBR::VARCHAR AS LINE_NBR, FILL_DT::DATE AS FILL_DT, - SYNTHETIC_DRUG_ID::VARCHAR, + SYNTHETIC_DRUG_ID::VARCHAR AS SYNTHETIC_DRUG_ID, GENERIC_DRUG_NAME::VARCHAR AS GENERIC_DRUG_NAME, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_commercial_pharmacy_2016.CSV', header=True, null_padding=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_commercial_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql index 846c430..5b4594d 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql @@ -14,93 +14,93 @@ SELECT TOB_CD::VARCHAR AS TOB_CD, CLM_TYPE_CD::VARCHAR AS CLM_TYPE_CD, DSCHRG_STUS::VARCHAR AS DSCHRG_STUS, - PRMRY_DX_IMPUTED::VARCHAR AS PRMRY_DX_IMPUTED, + PRMRY_DX_IMPUTED::NUMERIC AS PRMRY_DX_IMPUTED, PRMRY_DX_CD::VARCHAR AS PRMRY_DX_CD, ICD_DX_CD_1::VARCHAR AS ICD_DX_CD_1, ICD_DX_CD_2::VARCHAR AS ICD_DX_CD_2, - ICD_DX_CD_3::VARCHAR, - ICD_DX_CD_4::VARCHAR, - ICD_DX_CD_5::VARCHAR, - ICD_DX_CD_6::VARCHAR, - ICD_DX_CD_7::VARCHAR, - ICD_DX_CD_8::VARCHAR, - ICD_DX_CD_9::VARCHAR, - ICD_DX_CD_10::VARCHAR, - ICD_DX_CD_11::VARCHAR, - ICD_DX_CD_12::VARCHAR, - ICD_DX_CD_13::VARCHAR, - ICD_DX_CD_14::VARCHAR, - ICD_DX_CD_15::VARCHAR, - ICD_DX_CD_16::VARCHAR, - ICD_DX_CD_17::VARCHAR, - ICD_DX_CD_18::VARCHAR, - ICD_DX_CD_19::VARCHAR, - ICD_DX_CD_20::VARCHAR, - ICD_DX_CD_21::VARCHAR, - ICD_DX_CD_22::VARCHAR, - ICD_DX_CD_23::VARCHAR, - ICD_DX_CD_24::VARCHAR, + ICD_DX_CD_3::VARCHAR AS ICD_DX_CD_3, + ICD_DX_CD_4::VARCHAR AS ICD_DX_CD_4, + ICD_DX_CD_5::VARCHAR AS ICD_DX_CD_5, + ICD_DX_CD_6::VARCHAR AS ICD_DX_CD_6, + ICD_DX_CD_7::VARCHAR AS ICD_DX_CD_7, + ICD_DX_CD_8::VARCHAR AS ICD_DX_CD_8, + ICD_DX_CD_9::VARCHAR AS ICD_DX_CD_9, + ICD_DX_CD_10::VARCHAR AS ICD_DX_CD_10, + ICD_DX_CD_11::VARCHAR AS ICD_DX_CD_11, + ICD_DX_CD_12::VARCHAR AS ICD_DX_CD_12, + ICD_DX_CD_13::VARCHAR AS ICD_DX_CD_13, + ICD_DX_CD_14::VARCHAR AS ICD_DX_CD_14, + ICD_DX_CD_15::VARCHAR AS ICD_DX_CD_15, + ICD_DX_CD_16::VARCHAR AS ICD_DX_CD_16, + ICD_DX_CD_17::VARCHAR AS ICD_DX_CD_17, + ICD_DX_CD_18::VARCHAR AS ICD_DX_CD_18, + ICD_DX_CD_19::VARCHAR AS ICD_DX_CD_19, + ICD_DX_CD_20::VARCHAR AS ICD_DX_CD_20, + ICD_DX_CD_21::VARCHAR AS ICD_DX_CD_21, + ICD_DX_CD_22::VARCHAR AS ICD_DX_CD_22, + ICD_DX_CD_23::VARCHAR AS ICD_DX_CD_23, + ICD_DX_CD_24::VARCHAR AS ICD_DX_CD_24, ICD_DX_CD_25::VARCHAR AS ICD_DX_CD_25, ICD_PRCDR_CD_1::VARCHAR AS ICD_PRCDR_CD_1, ICD_PRCDR_CD_2::VARCHAR AS ICD_PRCDR_CD_2, - ICD_PRCDR_CD_3::VARCHAR, - ICD_PRCDR_CD_4::VARCHAR, - ICD_PRCDR_CD_5::VARCHAR, - ICD_PRCDR_CD_6::VARCHAR, - ICD_PRCDR_CD_7::VARCHAR, - ICD_PRCDR_CD_8::VARCHAR, - ICD_PRCDR_CD_9::VARCHAR, - ICD_PRCDR_CD_10::VARCHAR, - ICD_PRCDR_CD_11::VARCHAR, - ICD_PRCDR_CD_12::VARCHAR, - ICD_PRCDR_CD_13::VARCHAR, - ICD_PRCDR_CD_14::VARCHAR, - ICD_PRCDR_CD_15::VARCHAR, - ICD_PRCDR_CD_16::VARCHAR, - ICD_PRCDR_CD_17::VARCHAR, - ICD_PRCDR_CD_18::VARCHAR, - ICD_PRCDR_CD_19::VARCHAR, - ICD_PRCDR_CD_20::VARCHAR, - ICD_PRCDR_CD_21::VARCHAR, - ICD_PRCDR_CD_22::VARCHAR, - ICD_PRCDR_CD_23::VARCHAR, - ICD_PRCDR_CD_24::VARCHAR, + ICD_PRCDR_CD_3::VARCHAR AS ICD_PRCDR_CD_3, + ICD_PRCDR_CD_4::VARCHAR AS ICD_PRCDR_CD_4, + ICD_PRCDR_CD_5::VARCHAR AS ICD_PRCDR_CD_5, + ICD_PRCDR_CD_6::VARCHAR AS ICD_PRCDR_CD_6, + ICD_PRCDR_CD_7::VARCHAR AS ICD_PRCDR_CD_7, + ICD_PRCDR_CD_8::VARCHAR AS ICD_PRCDR_CD_8, + ICD_PRCDR_CD_9::VARCHAR AS ICD_PRCDR_CD_9, + ICD_PRCDR_CD_10::VARCHAR AS ICD_PRCDR_CD_10, + ICD_PRCDR_CD_11::VARCHAR AS ICD_PRCDR_CD_11, + ICD_PRCDR_CD_12::VARCHAR AS ICD_PRCDR_CD_12, + ICD_PRCDR_CD_13::VARCHAR AS ICD_PRCDR_CD_13, + ICD_PRCDR_CD_14::VARCHAR AS ICD_PRCDR_CD_14, + ICD_PRCDR_CD_15::VARCHAR AS ICD_PRCDR_CD_15, + ICD_PRCDR_CD_16::VARCHAR AS ICD_PRCDR_CD_16, + ICD_PRCDR_CD_17::VARCHAR AS ICD_PRCDR_CD_17, + ICD_PRCDR_CD_18::VARCHAR AS ICD_PRCDR_CD_18, + ICD_PRCDR_CD_19::VARCHAR AS ICD_PRCDR_CD_19, + ICD_PRCDR_CD_20::VARCHAR AS ICD_PRCDR_CD_20, + ICD_PRCDR_CD_21::VARCHAR AS ICD_PRCDR_CD_21, + ICD_PRCDR_CD_22::VARCHAR AS ICD_PRCDR_CD_22, + ICD_PRCDR_CD_23::VARCHAR AS ICD_PRCDR_CD_23, + ICD_PRCDR_CD_24::VARCHAR AS ICD_PRCDR_CD_24, ICD_PRCDR_CD_25::VARCHAR AS ICD_PRCDR_CD_25, CPT_PRCDR_CD_1::VARCHAR AS CPT_PRCDR_CD_1, - CPT_PRCDR_CD_2::VARCHAR, + CPT_PRCDR_CD_2::VARCHAR AS CPT_PRCDR_CD_2, CPT_PRCDR_CD_3::VARCHAR AS CPT_PRCDR_CD_3, - CPT_PRCDR_CD_4::VARCHAR, - CPT_PRCDR_CD_5::VARCHAR, - CPT_PRCDR_CD_6::VARCHAR, - CPT_PRCDR_CD_7::VARCHAR, - CPT_PRCDR_CD_8::VARCHAR, - CPT_PRCDR_CD_9::VARCHAR, - CPT_PRCDR_CD_10::VARCHAR, - CPT_PRCDR_CD_11::VARCHAR, - CPT_PRCDR_CD_12::VARCHAR, - CPT_PRCDR_CD_13::VARCHAR, - CPT_PRCDR_CD_14::VARCHAR, - CPT_PRCDR_CD_15::VARCHAR, - CPT_PRCDR_CD_16::VARCHAR, - CPT_PRCDR_CD_17::VARCHAR, - CPT_PRCDR_CD_18::VARCHAR, - CPT_PRCDR_CD_19::VARCHAR, - CPT_PRCDR_CD_20::VARCHAR, - CPT_PRCDR_CD_21::VARCHAR, - CPT_PRCDR_CD_22::VARCHAR, - CPT_PRCDR_CD_23::VARCHAR, - CPT_PRCDR_CD_24::VARCHAR, - CPT_PRCDR_CD_25::VARCHAR, - CPT_PRCDR_CD_26::VARCHAR, - CPT_PRCDR_CD_27::VARCHAR, - CPT_PRCDR_CD_28::VARCHAR, - CPT_PRCDR_CD_29::VARCHAR, - CPT_PRCDR_CD_30::VARCHAR, - CPT_PRCDR_CD_31::VARCHAR, - CPT_PRCDR_CD_32::VARCHAR, - CPT_PRCDR_CD_33::VARCHAR, - CPT_PRCDR_CD_34::VARCHAR, + CPT_PRCDR_CD_4::VARCHAR AS CPT_PRCDR_CD_4, + CPT_PRCDR_CD_5::VARCHAR AS CPT_PRCDR_CD_5, + CPT_PRCDR_CD_6::VARCHAR AS CPT_PRCDR_CD_6, + CPT_PRCDR_CD_7::VARCHAR AS CPT_PRCDR_CD_7, + CPT_PRCDR_CD_8::VARCHAR AS CPT_PRCDR_CD_8, + CPT_PRCDR_CD_9::VARCHAR AS CPT_PRCDR_CD_9, + CPT_PRCDR_CD_10::VARCHAR AS CPT_PRCDR_CD_10, + CPT_PRCDR_CD_11::VARCHAR AS CPT_PRCDR_CD_11, + CPT_PRCDR_CD_12::VARCHAR AS CPT_PRCDR_CD_12, + CPT_PRCDR_CD_13::VARCHAR AS CPT_PRCDR_CD_13, + CPT_PRCDR_CD_14::VARCHAR AS CPT_PRCDR_CD_14, + CPT_PRCDR_CD_15::VARCHAR AS CPT_PRCDR_CD_15, + CPT_PRCDR_CD_16::VARCHAR AS CPT_PRCDR_CD_16, + CPT_PRCDR_CD_17::VARCHAR AS CPT_PRCDR_CD_17, + CPT_PRCDR_CD_18::VARCHAR AS CPT_PRCDR_CD_18, + CPT_PRCDR_CD_19::VARCHAR AS CPT_PRCDR_CD_19, + CPT_PRCDR_CD_20::VARCHAR AS CPT_PRCDR_CD_20, + CPT_PRCDR_CD_21::VARCHAR AS CPT_PRCDR_CD_21, + CPT_PRCDR_CD_22::VARCHAR AS CPT_PRCDR_CD_22, + CPT_PRCDR_CD_23::VARCHAR AS CPT_PRCDR_CD_23, + CPT_PRCDR_CD_24::VARCHAR AS CPT_PRCDR_CD_24, + CPT_PRCDR_CD_25::VARCHAR AS CPT_PRCDR_CD_25, + CPT_PRCDR_CD_26::VARCHAR AS CPT_PRCDR_CD_26, + CPT_PRCDR_CD_27::VARCHAR AS CPT_PRCDR_CD_27, + CPT_PRCDR_CD_28::VARCHAR AS CPT_PRCDR_CD_28, + CPT_PRCDR_CD_29::VARCHAR AS CPT_PRCDR_CD_29, + CPT_PRCDR_CD_30::VARCHAR AS CPT_PRCDR_CD_30, + CPT_PRCDR_CD_31::VARCHAR AS CPT_PRCDR_CD_31, + CPT_PRCDR_CD_32::VARCHAR AS CPT_PRCDR_CD_32, + CPT_PRCDR_CD_33::VARCHAR AS CPT_PRCDR_CD_33, + CPT_PRCDR_CD_34::VARCHAR AS CPT_PRCDR_CD_34, CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicaid_inpatient_2016.CSV', header=True, null_padding=true, types={'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR'}, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicaid_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql index 5d95edc..95f8178 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql @@ -14,93 +14,93 @@ SELECT TOB_CD::VARCHAR AS TOB_CD, CLM_TYPE_CD::VARCHAR AS CLM_TYPE_CD, DSCHRG_STUS::VARCHAR AS DSCHRG_STUS, - PRMRY_DX_IMPUTED::VARCHAR AS PRMRY_DX_IMPUTED, + PRMRY_DX_IMPUTED::NUMERIC AS PRMRY_DX_IMPUTED, PRMRY_DX_CD::VARCHAR AS PRMRY_DX_CD, ICD_DX_CD_1::VARCHAR AS ICD_DX_CD_1, ICD_DX_CD_2::VARCHAR AS ICD_DX_CD_2, - ICD_DX_CD_3::VARCHAR, - ICD_DX_CD_4::VARCHAR, - ICD_DX_CD_5::VARCHAR, - ICD_DX_CD_6::VARCHAR, - ICD_DX_CD_7::VARCHAR, - ICD_DX_CD_8::VARCHAR, - ICD_DX_CD_9::VARCHAR, - ICD_DX_CD_10::VARCHAR, - ICD_DX_CD_11::VARCHAR, - ICD_DX_CD_12::VARCHAR, - ICD_DX_CD_13::VARCHAR, - ICD_DX_CD_14::VARCHAR, - ICD_DX_CD_15::VARCHAR, - ICD_DX_CD_16::VARCHAR, - ICD_DX_CD_17::VARCHAR, - ICD_DX_CD_18::VARCHAR, - ICD_DX_CD_19::VARCHAR, - ICD_DX_CD_20::VARCHAR, - ICD_DX_CD_21::VARCHAR, - ICD_DX_CD_22::VARCHAR, - ICD_DX_CD_23::VARCHAR, - ICD_DX_CD_24::VARCHAR, + ICD_DX_CD_3::VARCHAR AS ICD_DX_CD_3, + ICD_DX_CD_4::VARCHAR AS ICD_DX_CD_4, + ICD_DX_CD_5::VARCHAR AS ICD_DX_CD_5, + ICD_DX_CD_6::VARCHAR AS ICD_DX_CD_6, + ICD_DX_CD_7::VARCHAR AS ICD_DX_CD_7, + ICD_DX_CD_8::VARCHAR AS ICD_DX_CD_8, + ICD_DX_CD_9::VARCHAR AS ICD_DX_CD_9, + ICD_DX_CD_10::VARCHAR AS ICD_DX_CD_10, + ICD_DX_CD_11::VARCHAR AS ICD_DX_CD_11, + ICD_DX_CD_12::VARCHAR AS ICD_DX_CD_12, + ICD_DX_CD_13::VARCHAR AS ICD_DX_CD_13, + ICD_DX_CD_14::VARCHAR AS ICD_DX_CD_14, + ICD_DX_CD_15::VARCHAR AS ICD_DX_CD_15, + ICD_DX_CD_16::VARCHAR AS ICD_DX_CD_16, + ICD_DX_CD_17::VARCHAR AS ICD_DX_CD_17, + ICD_DX_CD_18::VARCHAR AS ICD_DX_CD_18, + ICD_DX_CD_19::VARCHAR AS ICD_DX_CD_19, + ICD_DX_CD_20::VARCHAR AS ICD_DX_CD_20, + ICD_DX_CD_21::VARCHAR AS ICD_DX_CD_21, + ICD_DX_CD_22::VARCHAR AS ICD_DX_CD_22, + ICD_DX_CD_23::VARCHAR AS ICD_DX_CD_23, + ICD_DX_CD_24::VARCHAR AS ICD_DX_CD_24, ICD_DX_CD_25::VARCHAR AS ICD_DX_CD_25, ICD_PRCDR_CD_1::VARCHAR AS ICD_PRCDR_CD_1, ICD_PRCDR_CD_2::VARCHAR AS ICD_PRCDR_CD_2, - ICD_PRCDR_CD_3::VARCHAR, - ICD_PRCDR_CD_4::VARCHAR, - ICD_PRCDR_CD_5::VARCHAR, - ICD_PRCDR_CD_6::VARCHAR, - ICD_PRCDR_CD_7::VARCHAR, - ICD_PRCDR_CD_8::VARCHAR, - ICD_PRCDR_CD_9::VARCHAR, - ICD_PRCDR_CD_10::VARCHAR, - ICD_PRCDR_CD_11::VARCHAR, - ICD_PRCDR_CD_12::VARCHAR, - ICD_PRCDR_CD_13::VARCHAR, - ICD_PRCDR_CD_14::VARCHAR, - ICD_PRCDR_CD_15::VARCHAR, - ICD_PRCDR_CD_16::VARCHAR, - ICD_PRCDR_CD_17::VARCHAR, - ICD_PRCDR_CD_18::VARCHAR, - ICD_PRCDR_CD_19::VARCHAR, - ICD_PRCDR_CD_20::VARCHAR, - ICD_PRCDR_CD_21::VARCHAR, - ICD_PRCDR_CD_22::VARCHAR, - ICD_PRCDR_CD_23::VARCHAR, - ICD_PRCDR_CD_24::VARCHAR, + ICD_PRCDR_CD_3::VARCHAR AS ICD_PRCDR_CD_3, + ICD_PRCDR_CD_4::VARCHAR AS ICD_PRCDR_CD_4, + ICD_PRCDR_CD_5::VARCHAR AS ICD_PRCDR_CD_5, + ICD_PRCDR_CD_6::VARCHAR AS ICD_PRCDR_CD_6, + ICD_PRCDR_CD_7::VARCHAR AS ICD_PRCDR_CD_7, + ICD_PRCDR_CD_8::VARCHAR AS ICD_PRCDR_CD_8, + ICD_PRCDR_CD_9::VARCHAR AS ICD_PRCDR_CD_9, + ICD_PRCDR_CD_10::VARCHAR AS ICD_PRCDR_CD_10, + ICD_PRCDR_CD_11::VARCHAR AS ICD_PRCDR_CD_11, + ICD_PRCDR_CD_12::VARCHAR AS ICD_PRCDR_CD_12, + ICD_PRCDR_CD_13::VARCHAR AS ICD_PRCDR_CD_13, + ICD_PRCDR_CD_14::VARCHAR AS ICD_PRCDR_CD_14, + ICD_PRCDR_CD_15::VARCHAR AS ICD_PRCDR_CD_15, + ICD_PRCDR_CD_16::VARCHAR AS ICD_PRCDR_CD_16, + ICD_PRCDR_CD_17::VARCHAR AS ICD_PRCDR_CD_17, + ICD_PRCDR_CD_18::VARCHAR AS ICD_PRCDR_CD_18, + ICD_PRCDR_CD_19::VARCHAR AS ICD_PRCDR_CD_19, + ICD_PRCDR_CD_20::VARCHAR AS ICD_PRCDR_CD_20, + ICD_PRCDR_CD_21::VARCHAR AS ICD_PRCDR_CD_21, + ICD_PRCDR_CD_22::VARCHAR AS ICD_PRCDR_CD_22, + ICD_PRCDR_CD_23::VARCHAR AS ICD_PRCDR_CD_23, + ICD_PRCDR_CD_24::VARCHAR AS ICD_PRCDR_CD_24, ICD_PRCDR_CD_25::VARCHAR AS ICD_PRCDR_CD_25, CPT_PRCDR_CD_1::VARCHAR AS CPT_PRCDR_CD_1, - CPT_PRCDR_CD_2::VARCHAR, + CPT_PRCDR_CD_2::VARCHAR AS CPT_PRCDR_CD_2, CPT_PRCDR_CD_3::VARCHAR AS CPT_PRCDR_CD_3, - CPT_PRCDR_CD_4::VARCHAR, - CPT_PRCDR_CD_5::VARCHAR, - CPT_PRCDR_CD_6::VARCHAR, - CPT_PRCDR_CD_7::VARCHAR, - CPT_PRCDR_CD_8::VARCHAR, - CPT_PRCDR_CD_9::VARCHAR, - CPT_PRCDR_CD_10::VARCHAR, - CPT_PRCDR_CD_11::VARCHAR, - CPT_PRCDR_CD_12::VARCHAR, - CPT_PRCDR_CD_13::VARCHAR, - CPT_PRCDR_CD_14::VARCHAR, - CPT_PRCDR_CD_15::VARCHAR, - CPT_PRCDR_CD_16::VARCHAR, - CPT_PRCDR_CD_17::VARCHAR, - CPT_PRCDR_CD_18::VARCHAR, - CPT_PRCDR_CD_19::VARCHAR, - CPT_PRCDR_CD_20::VARCHAR, - CPT_PRCDR_CD_21::VARCHAR, - CPT_PRCDR_CD_22::VARCHAR, - CPT_PRCDR_CD_23::VARCHAR, - CPT_PRCDR_CD_24::VARCHAR, - CPT_PRCDR_CD_25::VARCHAR, - CPT_PRCDR_CD_26::VARCHAR, - CPT_PRCDR_CD_27::VARCHAR, - CPT_PRCDR_CD_28::VARCHAR, - CPT_PRCDR_CD_29::VARCHAR, - CPT_PRCDR_CD_30::VARCHAR, - CPT_PRCDR_CD_31::VARCHAR, - CPT_PRCDR_CD_32::VARCHAR, - CPT_PRCDR_CD_33::VARCHAR, - CPT_PRCDR_CD_34::VARCHAR, + CPT_PRCDR_CD_4::VARCHAR AS CPT_PRCDR_CD_4, + CPT_PRCDR_CD_5::VARCHAR AS CPT_PRCDR_CD_5, + CPT_PRCDR_CD_6::VARCHAR AS CPT_PRCDR_CD_6, + CPT_PRCDR_CD_7::VARCHAR AS CPT_PRCDR_CD_7, + CPT_PRCDR_CD_8::VARCHAR AS CPT_PRCDR_CD_8, + CPT_PRCDR_CD_9::VARCHAR AS CPT_PRCDR_CD_9, + CPT_PRCDR_CD_10::VARCHAR AS CPT_PRCDR_CD_10, + CPT_PRCDR_CD_11::VARCHAR AS CPT_PRCDR_CD_11, + CPT_PRCDR_CD_12::VARCHAR AS CPT_PRCDR_CD_12, + CPT_PRCDR_CD_13::VARCHAR AS CPT_PRCDR_CD_13, + CPT_PRCDR_CD_14::VARCHAR AS CPT_PRCDR_CD_14, + CPT_PRCDR_CD_15::VARCHAR AS CPT_PRCDR_CD_15, + CPT_PRCDR_CD_16::VARCHAR AS CPT_PRCDR_CD_16, + CPT_PRCDR_CD_17::VARCHAR AS CPT_PRCDR_CD_17, + CPT_PRCDR_CD_18::VARCHAR AS CPT_PRCDR_CD_18, + CPT_PRCDR_CD_19::VARCHAR AS CPT_PRCDR_CD_19, + CPT_PRCDR_CD_20::VARCHAR AS CPT_PRCDR_CD_20, + CPT_PRCDR_CD_21::VARCHAR AS CPT_PRCDR_CD_21, + CPT_PRCDR_CD_22::VARCHAR AS CPT_PRCDR_CD_22, + CPT_PRCDR_CD_23::VARCHAR AS CPT_PRCDR_CD_23, + CPT_PRCDR_CD_24::VARCHAR AS CPT_PRCDR_CD_24, + CPT_PRCDR_CD_25::VARCHAR AS CPT_PRCDR_CD_25, + CPT_PRCDR_CD_26::VARCHAR AS CPT_PRCDR_CD_26, + CPT_PRCDR_CD_27::VARCHAR AS CPT_PRCDR_CD_27, + CPT_PRCDR_CD_28::VARCHAR AS CPT_PRCDR_CD_28, + CPT_PRCDR_CD_29::VARCHAR AS CPT_PRCDR_CD_29, + CPT_PRCDR_CD_30::VARCHAR AS CPT_PRCDR_CD_30, + CPT_PRCDR_CD_31::VARCHAR AS CPT_PRCDR_CD_31, + CPT_PRCDR_CD_32::VARCHAR AS CPT_PRCDR_CD_32, + CPT_PRCDR_CD_33::VARCHAR AS CPT_PRCDR_CD_33, + CPT_PRCDR_CD_34::VARCHAR AS CPT_PRCDR_CD_34, CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicaid_outpatient_2016.CSV', header=True, null_padding=true, types={'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR'}, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicaid_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_person_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_person_2016.sql index 9908a43..8b3adfa 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_person_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_person_2016.sql @@ -7,7 +7,7 @@ SELECT PERSON_WGHT::NUMERIC AS PERSON_WGHT, AGE_LOW::NUMERIC AS AGE_LOW, AGE_HIGH::NUMERIC AS AGE_HIGH, - SEX_IDENT_CD::VARCHAR AS SEX_IDENT_CD, + SEX_IDENT_CD::UBIGINT AS SEX_IDENT_CD, RACE_CD::VARCHAR AS RACE_CD, MCAID_SBMTTG_ST_CD::VARCHAR AS MCAID_SBMTTG_ST_CD, STATE_CD::VARCHAR AS STATE_CD, @@ -15,41 +15,41 @@ SELECT ZIP_CD::VARCHAR AS ZIP_CD, RSN_ENRLMT_CD::VARCHAR AS RSN_ENRLMT_CD, MDCD_ENRLMT_1::NUMERIC AS MDCD_ENRLMT_1, - MDCD_ENRLMT_2::VARCHAR, - MDCD_ENRLMT_3::VARCHAR, - MDCD_ENRLMT_4::VARCHAR, - MDCD_ENRLMT_5::VARCHAR, - MDCD_ENRLMT_6::VARCHAR, - MDCD_ENRLMT_7::VARCHAR, - MDCD_ENRLMT_8::VARCHAR, - MDCD_ENRLMT_9::VARCHAR, - MDCD_ENRLMT_10::VARCHAR, - MDCD_ENRLMT_11::VARCHAR, + MDCD_ENRLMT_2::NUMERIC AS MDCD_ENRLMT_2, + MDCD_ENRLMT_3::NUMERIC AS MDCD_ENRLMT_3, + MDCD_ENRLMT_4::NUMERIC AS MDCD_ENRLMT_4, + MDCD_ENRLMT_5::NUMERIC AS MDCD_ENRLMT_5, + MDCD_ENRLMT_6::NUMERIC AS MDCD_ENRLMT_6, + MDCD_ENRLMT_7::NUMERIC AS MDCD_ENRLMT_7, + MDCD_ENRLMT_8::NUMERIC AS MDCD_ENRLMT_8, + MDCD_ENRLMT_9::NUMERIC AS MDCD_ENRLMT_9, + MDCD_ENRLMT_10::NUMERIC AS MDCD_ENRLMT_10, + MDCD_ENRLMT_11::NUMERIC AS MDCD_ENRLMT_11, MDCD_ENRLMT_12::NUMERIC AS MDCD_ENRLMT_12, MDCD_MCO_ENRLMT_1::NUMERIC AS MDCD_MCO_ENRLMT_1, - MDCD_MCO_ENRLMT_2::VARCHAR, - MDCD_MCO_ENRLMT_3::VARCHAR, - MDCD_MCO_ENRLMT_4::VARCHAR, - MDCD_MCO_ENRLMT_5::VARCHAR, - MDCD_MCO_ENRLMT_6::VARCHAR, - MDCD_MCO_ENRLMT_7::VARCHAR, - MDCD_MCO_ENRLMT_8::VARCHAR, - MDCD_MCO_ENRLMT_9::VARCHAR, - MDCD_MCO_ENRLMT_10::VARCHAR, - MDCD_MCO_ENRLMT_11::VARCHAR, + MDCD_MCO_ENRLMT_2::NUMERIC AS MDCD_MCO_ENRLMT_2, + MDCD_MCO_ENRLMT_3::NUMERIC AS MDCD_MCO_ENRLMT_3, + MDCD_MCO_ENRLMT_4::NUMERIC AS MDCD_MCO_ENRLMT_4, + MDCD_MCO_ENRLMT_5::NUMERIC AS MDCD_MCO_ENRLMT_5, + MDCD_MCO_ENRLMT_6::NUMERIC AS MDCD_MCO_ENRLMT_6, + MDCD_MCO_ENRLMT_7::NUMERIC AS MDCD_MCO_ENRLMT_7, + MDCD_MCO_ENRLMT_8::NUMERIC AS MDCD_MCO_ENRLMT_8, + MDCD_MCO_ENRLMT_9::NUMERIC AS MDCD_MCO_ENRLMT_9, + MDCD_MCO_ENRLMT_10::NUMERIC AS MDCD_MCO_ENRLMT_10, + MDCD_MCO_ENRLMT_11::NUMERIC AS MDCD_MCO_ENRLMT_11, MDCD_MCO_ENRLMT_12::NUMERIC AS MDCD_MCO_ENRLMT_12, MDCD_CHIP_ENRLMT::NUMERIC AS MDCD_CHIP_ENRLMT, - RSTRCTD_BNFTS_IND::VARCHAR, + RSTRCTD_BNFTS_IND::NUMERIC AS RSTRCTD_BNFTS_IND, DUAL_ELGBL_1::NUMERIC AS DUAL_ELGBL_1, - DUAL_ELGBL_2::VARCHAR, - DUAL_ELGBL_3::VARCHAR, - DUAL_ELGBL_4::VARCHAR, - DUAL_ELGBL_5::VARCHAR, - DUAL_ELGBL_6::VARCHAR, - DUAL_ELGBL_7::VARCHAR, - DUAL_ELGBL_8::VARCHAR, - DUAL_ELGBL_9::VARCHAR, - DUAL_ELGBL_10::VARCHAR, - DUAL_ELGBL_11::VARCHAR, + DUAL_ELGBL_2::NUMERIC AS DUAL_ELGBL_2, + DUAL_ELGBL_3::NUMERIC AS DUAL_ELGBL_3, + DUAL_ELGBL_4::NUMERIC AS DUAL_ELGBL_4, + DUAL_ELGBL_5::NUMERIC AS DUAL_ELGBL_5, + DUAL_ELGBL_6::NUMERIC AS DUAL_ELGBL_6, + DUAL_ELGBL_7::NUMERIC AS DUAL_ELGBL_7, + DUAL_ELGBL_8::NUMERIC AS DUAL_ELGBL_8, + DUAL_ELGBL_9::NUMERIC AS DUAL_ELGBL_9, + DUAL_ELGBL_10::NUMERIC AS DUAL_ELGBL_10, + DUAL_ELGBL_11::NUMERIC AS DUAL_ELGBL_11, DUAL_ELGBL_12::NUMERIC AS DUAL_ELGBL_12 -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicaid_person_2016.CSV', header=True, null_padding=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicaid_person_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'MCAID_BENE_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'AGE_LOW': 'NUMERIC', 'AGE_HIGH': 'NUMERIC', 'SEX_IDENT_CD': 'UBIGINT', 'RACE_CD': 'VARCHAR', 'MCAID_SBMTTG_ST_CD': 'VARCHAR', 'STATE_CD': 'VARCHAR', 'COUNTY_FIPS_CD': 'VARCHAR', 'ZIP_CD': 'VARCHAR', 'RSN_ENRLMT_CD': 'VARCHAR', 'MDCD_ENRLMT_1': 'NUMERIC', 'MDCD_ENRLMT_2': 'NUMERIC', 'MDCD_ENRLMT_3': 'NUMERIC', 'MDCD_ENRLMT_4': 'NUMERIC', 'MDCD_ENRLMT_5': 'NUMERIC', 'MDCD_ENRLMT_6': 'NUMERIC', 'MDCD_ENRLMT_7': 'NUMERIC', 'MDCD_ENRLMT_8': 'NUMERIC', 'MDCD_ENRLMT_9': 'NUMERIC', 'MDCD_ENRLMT_10': 'NUMERIC', 'MDCD_ENRLMT_11': 'NUMERIC', 'MDCD_ENRLMT_12': 'NUMERIC', 'MDCD_MCO_ENRLMT_1': 'NUMERIC', 'MDCD_MCO_ENRLMT_2': 'NUMERIC', 'MDCD_MCO_ENRLMT_3': 'NUMERIC', 'MDCD_MCO_ENRLMT_4': 'NUMERIC', 'MDCD_MCO_ENRLMT_5': 'NUMERIC', 'MDCD_MCO_ENRLMT_6': 'NUMERIC', 'MDCD_MCO_ENRLMT_7': 'NUMERIC', 'MDCD_MCO_ENRLMT_8': 'NUMERIC', 'MDCD_MCO_ENRLMT_9': 'NUMERIC', 'MDCD_MCO_ENRLMT_10': 'NUMERIC', 'MDCD_MCO_ENRLMT_11': 'NUMERIC', 'MDCD_MCO_ENRLMT_12': 'NUMERIC', 'MDCD_CHIP_ENRLMT': 'NUMERIC', 'RSTRCTD_BNFTS_IND': 'NUMERIC', 'DUAL_ELGBL_1': 'NUMERIC', 'DUAL_ELGBL_2': 'NUMERIC', 'DUAL_ELGBL_3': 'NUMERIC', 'DUAL_ELGBL_4': 'NUMERIC', 'DUAL_ELGBL_5': 'NUMERIC', 'DUAL_ELGBL_6': 'NUMERIC', 'DUAL_ELGBL_7': 'NUMERIC', 'DUAL_ELGBL_8': 'NUMERIC', 'DUAL_ELGBL_9': 'NUMERIC', 'DUAL_ELGBL_10': 'NUMERIC', 'DUAL_ELGBL_11': 'NUMERIC', 'DUAL_ELGBL_12': 'NUMERIC' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql index cf842f4..d2cff95 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql @@ -6,10 +6,10 @@ SELECT PERSON_WGHT::NUMERIC AS PERSON_WGHT, PHMCY_CLM_NUM::NUMERIC AS PHMCY_CLM_NUM, CLM_CNTL_NUM::NUMERIC AS CLM_CNTL_NUM, - LINE_NBR::VARCHAR, + LINE_NBR::VARCHAR AS LINE_NBR, FILL_DT::DATE AS FILL_DT, - SYNTHETIC_DRUG_ID::VARCHAR, + SYNTHETIC_DRUG_ID::VARCHAR AS SYNTHETIC_DRUG_ID, GENERIC_DRUG_NAME::VARCHAR AS GENERIC_DRUG_NAME, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicaid_pharmacy_2016.CSV', header=True, null_padding=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicaid_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_provider_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_provider_2016.sql index 93dcc91..56353b1 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_provider_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_provider_2016.sql @@ -1,9 +1,9 @@ --- SQL model for syhdr_medicaid_provider_2016.csv +-- SQL model for syhdr_medicaid_provider_2016.CSV {{ config(materialized='external', location=var('output_path') + '/' + this.name + '.parquet') }} SELECT - Facility_ID::VARCHAR, - Prvdr_Ctgry_Cd::VARCHAR, - Prvdr_Ownrshp_Cd::VARCHAR, - Prvdr_Prtcptn_Cd::VARCHAR -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicaid_provider_2016.csv', header=True, null_padding=true) \ No newline at end of file + Facility_ID::VARCHAR AS Facility_ID, + Prvdr_Ctgry_Cd::VARCHAR AS Prvdr_Ctgry_Cd, + Prvdr_Ownrshp_Cd::VARCHAR AS Prvdr_Ownrshp_Cd, + Prvdr_Prtcptn_Cd::VARCHAR AS Prvdr_Prtcptn_Cd +FROM read_csv('~/data/syh_dr/syhdr_medicaid_provider_2016.CSV', header=True, null_padding=true, types={ 'Facility_ID': 'VARCHAR', 'Prvdr_Ctgry_Cd': 'VARCHAR', 'Prvdr_Ownrshp_Cd': 'VARCHAR', 'Prvdr_Prtcptn_Cd': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql index ed0e957..fd2ad3f 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql @@ -14,93 +14,93 @@ SELECT TOB_CD::VARCHAR AS TOB_CD, CLM_TYPE_CD::VARCHAR AS CLM_TYPE_CD, DSCHRG_STUS::VARCHAR AS DSCHRG_STUS, - PRMRY_DX_IMPUTED::VARCHAR AS PRMRY_DX_IMPUTED, + PRMRY_DX_IMPUTED::NUMERIC AS PRMRY_DX_IMPUTED, PRMRY_DX_CD::VARCHAR AS PRMRY_DX_CD, ICD_DX_CD_1::VARCHAR AS ICD_DX_CD_1, ICD_DX_CD_2::VARCHAR AS ICD_DX_CD_2, - ICD_DX_CD_3::VARCHAR, - ICD_DX_CD_4::VARCHAR, - ICD_DX_CD_5::VARCHAR, - ICD_DX_CD_6::VARCHAR, - ICD_DX_CD_7::VARCHAR, - ICD_DX_CD_8::VARCHAR, - ICD_DX_CD_9::VARCHAR, - ICD_DX_CD_10::VARCHAR, - ICD_DX_CD_11::VARCHAR, - ICD_DX_CD_12::VARCHAR, - ICD_DX_CD_13::VARCHAR, - ICD_DX_CD_14::VARCHAR, - ICD_DX_CD_15::VARCHAR, - ICD_DX_CD_16::VARCHAR, - ICD_DX_CD_17::VARCHAR, - ICD_DX_CD_18::VARCHAR, - ICD_DX_CD_19::VARCHAR, - ICD_DX_CD_20::VARCHAR, - ICD_DX_CD_21::VARCHAR, - ICD_DX_CD_22::VARCHAR, - ICD_DX_CD_23::VARCHAR, - ICD_DX_CD_24::VARCHAR, + ICD_DX_CD_3::VARCHAR AS ICD_DX_CD_3, + ICD_DX_CD_4::VARCHAR AS ICD_DX_CD_4, + ICD_DX_CD_5::VARCHAR AS ICD_DX_CD_5, + ICD_DX_CD_6::VARCHAR AS ICD_DX_CD_6, + ICD_DX_CD_7::VARCHAR AS ICD_DX_CD_7, + ICD_DX_CD_8::VARCHAR AS ICD_DX_CD_8, + ICD_DX_CD_9::VARCHAR AS ICD_DX_CD_9, + ICD_DX_CD_10::VARCHAR AS ICD_DX_CD_10, + ICD_DX_CD_11::VARCHAR AS ICD_DX_CD_11, + ICD_DX_CD_12::VARCHAR AS ICD_DX_CD_12, + ICD_DX_CD_13::VARCHAR AS ICD_DX_CD_13, + ICD_DX_CD_14::VARCHAR AS ICD_DX_CD_14, + ICD_DX_CD_15::VARCHAR AS ICD_DX_CD_15, + ICD_DX_CD_16::VARCHAR AS ICD_DX_CD_16, + ICD_DX_CD_17::VARCHAR AS ICD_DX_CD_17, + ICD_DX_CD_18::VARCHAR AS ICD_DX_CD_18, + ICD_DX_CD_19::VARCHAR AS ICD_DX_CD_19, + ICD_DX_CD_20::VARCHAR AS ICD_DX_CD_20, + ICD_DX_CD_21::VARCHAR AS ICD_DX_CD_21, + ICD_DX_CD_22::VARCHAR AS ICD_DX_CD_22, + ICD_DX_CD_23::VARCHAR AS ICD_DX_CD_23, + ICD_DX_CD_24::VARCHAR AS ICD_DX_CD_24, ICD_DX_CD_25::VARCHAR AS ICD_DX_CD_25, ICD_PRCDR_CD_1::VARCHAR AS ICD_PRCDR_CD_1, ICD_PRCDR_CD_2::VARCHAR AS ICD_PRCDR_CD_2, - ICD_PRCDR_CD_3::VARCHAR, - ICD_PRCDR_CD_4::VARCHAR, - ICD_PRCDR_CD_5::VARCHAR, - ICD_PRCDR_CD_6::VARCHAR, - ICD_PRCDR_CD_7::VARCHAR, - ICD_PRCDR_CD_8::VARCHAR, - ICD_PRCDR_CD_9::VARCHAR, - ICD_PRCDR_CD_10::VARCHAR, - ICD_PRCDR_CD_11::VARCHAR, - ICD_PRCDR_CD_12::VARCHAR, - ICD_PRCDR_CD_13::VARCHAR, - ICD_PRCDR_CD_14::VARCHAR, - ICD_PRCDR_CD_15::VARCHAR, - ICD_PRCDR_CD_16::VARCHAR, - ICD_PRCDR_CD_17::VARCHAR, - ICD_PRCDR_CD_18::VARCHAR, - ICD_PRCDR_CD_19::VARCHAR, - ICD_PRCDR_CD_20::VARCHAR, - ICD_PRCDR_CD_21::VARCHAR, - ICD_PRCDR_CD_22::VARCHAR, - ICD_PRCDR_CD_23::VARCHAR, - ICD_PRCDR_CD_24::VARCHAR, + ICD_PRCDR_CD_3::VARCHAR AS ICD_PRCDR_CD_3, + ICD_PRCDR_CD_4::VARCHAR AS ICD_PRCDR_CD_4, + ICD_PRCDR_CD_5::VARCHAR AS ICD_PRCDR_CD_5, + ICD_PRCDR_CD_6::VARCHAR AS ICD_PRCDR_CD_6, + ICD_PRCDR_CD_7::VARCHAR AS ICD_PRCDR_CD_7, + ICD_PRCDR_CD_8::VARCHAR AS ICD_PRCDR_CD_8, + ICD_PRCDR_CD_9::VARCHAR AS ICD_PRCDR_CD_9, + ICD_PRCDR_CD_10::VARCHAR AS ICD_PRCDR_CD_10, + ICD_PRCDR_CD_11::VARCHAR AS ICD_PRCDR_CD_11, + ICD_PRCDR_CD_12::VARCHAR AS ICD_PRCDR_CD_12, + ICD_PRCDR_CD_13::VARCHAR AS ICD_PRCDR_CD_13, + ICD_PRCDR_CD_14::VARCHAR AS ICD_PRCDR_CD_14, + ICD_PRCDR_CD_15::VARCHAR AS ICD_PRCDR_CD_15, + ICD_PRCDR_CD_16::VARCHAR AS ICD_PRCDR_CD_16, + ICD_PRCDR_CD_17::VARCHAR AS ICD_PRCDR_CD_17, + ICD_PRCDR_CD_18::VARCHAR AS ICD_PRCDR_CD_18, + ICD_PRCDR_CD_19::VARCHAR AS ICD_PRCDR_CD_19, + ICD_PRCDR_CD_20::VARCHAR AS ICD_PRCDR_CD_20, + ICD_PRCDR_CD_21::VARCHAR AS ICD_PRCDR_CD_21, + ICD_PRCDR_CD_22::VARCHAR AS ICD_PRCDR_CD_22, + ICD_PRCDR_CD_23::VARCHAR AS ICD_PRCDR_CD_23, + ICD_PRCDR_CD_24::VARCHAR AS ICD_PRCDR_CD_24, ICD_PRCDR_CD_25::VARCHAR AS ICD_PRCDR_CD_25, CPT_PRCDR_CD_1::VARCHAR AS CPT_PRCDR_CD_1, - CPT_PRCDR_CD_2::VARCHAR, + CPT_PRCDR_CD_2::VARCHAR AS CPT_PRCDR_CD_2, CPT_PRCDR_CD_3::VARCHAR AS CPT_PRCDR_CD_3, - CPT_PRCDR_CD_4::VARCHAR, - CPT_PRCDR_CD_5::VARCHAR, - CPT_PRCDR_CD_6::VARCHAR, - CPT_PRCDR_CD_7::VARCHAR, - CPT_PRCDR_CD_8::VARCHAR, - CPT_PRCDR_CD_9::VARCHAR, - CPT_PRCDR_CD_10::VARCHAR, - CPT_PRCDR_CD_11::VARCHAR, - CPT_PRCDR_CD_12::VARCHAR, - CPT_PRCDR_CD_13::VARCHAR, - CPT_PRCDR_CD_14::VARCHAR, - CPT_PRCDR_CD_15::VARCHAR, - CPT_PRCDR_CD_16::VARCHAR, - CPT_PRCDR_CD_17::VARCHAR, - CPT_PRCDR_CD_18::VARCHAR, - CPT_PRCDR_CD_19::VARCHAR, - CPT_PRCDR_CD_20::VARCHAR, - CPT_PRCDR_CD_21::VARCHAR, - CPT_PRCDR_CD_22::VARCHAR, - CPT_PRCDR_CD_23::VARCHAR, - CPT_PRCDR_CD_24::VARCHAR, - CPT_PRCDR_CD_25::VARCHAR, - CPT_PRCDR_CD_26::VARCHAR, - CPT_PRCDR_CD_27::VARCHAR, - CPT_PRCDR_CD_28::VARCHAR, - CPT_PRCDR_CD_29::VARCHAR, - CPT_PRCDR_CD_30::VARCHAR, - CPT_PRCDR_CD_31::VARCHAR, - CPT_PRCDR_CD_32::VARCHAR, - CPT_PRCDR_CD_33::VARCHAR, - CPT_PRCDR_CD_34::VARCHAR, + CPT_PRCDR_CD_4::VARCHAR AS CPT_PRCDR_CD_4, + CPT_PRCDR_CD_5::VARCHAR AS CPT_PRCDR_CD_5, + CPT_PRCDR_CD_6::VARCHAR AS CPT_PRCDR_CD_6, + CPT_PRCDR_CD_7::VARCHAR AS CPT_PRCDR_CD_7, + CPT_PRCDR_CD_8::VARCHAR AS CPT_PRCDR_CD_8, + CPT_PRCDR_CD_9::VARCHAR AS CPT_PRCDR_CD_9, + CPT_PRCDR_CD_10::VARCHAR AS CPT_PRCDR_CD_10, + CPT_PRCDR_CD_11::VARCHAR AS CPT_PRCDR_CD_11, + CPT_PRCDR_CD_12::VARCHAR AS CPT_PRCDR_CD_12, + CPT_PRCDR_CD_13::VARCHAR AS CPT_PRCDR_CD_13, + CPT_PRCDR_CD_14::VARCHAR AS CPT_PRCDR_CD_14, + CPT_PRCDR_CD_15::VARCHAR AS CPT_PRCDR_CD_15, + CPT_PRCDR_CD_16::VARCHAR AS CPT_PRCDR_CD_16, + CPT_PRCDR_CD_17::VARCHAR AS CPT_PRCDR_CD_17, + CPT_PRCDR_CD_18::VARCHAR AS CPT_PRCDR_CD_18, + CPT_PRCDR_CD_19::VARCHAR AS CPT_PRCDR_CD_19, + CPT_PRCDR_CD_20::VARCHAR AS CPT_PRCDR_CD_20, + CPT_PRCDR_CD_21::VARCHAR AS CPT_PRCDR_CD_21, + CPT_PRCDR_CD_22::VARCHAR AS CPT_PRCDR_CD_22, + CPT_PRCDR_CD_23::VARCHAR AS CPT_PRCDR_CD_23, + CPT_PRCDR_CD_24::VARCHAR AS CPT_PRCDR_CD_24, + CPT_PRCDR_CD_25::VARCHAR AS CPT_PRCDR_CD_25, + CPT_PRCDR_CD_26::VARCHAR AS CPT_PRCDR_CD_26, + CPT_PRCDR_CD_27::VARCHAR AS CPT_PRCDR_CD_27, + CPT_PRCDR_CD_28::VARCHAR AS CPT_PRCDR_CD_28, + CPT_PRCDR_CD_29::VARCHAR AS CPT_PRCDR_CD_29, + CPT_PRCDR_CD_30::VARCHAR AS CPT_PRCDR_CD_30, + CPT_PRCDR_CD_31::VARCHAR AS CPT_PRCDR_CD_31, + CPT_PRCDR_CD_32::VARCHAR AS CPT_PRCDR_CD_32, + CPT_PRCDR_CD_33::VARCHAR AS CPT_PRCDR_CD_33, + CPT_PRCDR_CD_34::VARCHAR AS CPT_PRCDR_CD_34, CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicare_inpatient_2016.CSV', header=True, null_padding=true, types={'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR'}, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicare_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql index 0eb0ff3..77f58f0 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql @@ -14,93 +14,93 @@ SELECT TOB_CD::VARCHAR AS TOB_CD, CLM_TYPE_CD::VARCHAR AS CLM_TYPE_CD, DSCHRG_STUS::VARCHAR AS DSCHRG_STUS, - PRMRY_DX_IMPUTED::VARCHAR AS PRMRY_DX_IMPUTED, + PRMRY_DX_IMPUTED::NUMERIC AS PRMRY_DX_IMPUTED, PRMRY_DX_CD::VARCHAR AS PRMRY_DX_CD, ICD_DX_CD_1::VARCHAR AS ICD_DX_CD_1, ICD_DX_CD_2::VARCHAR AS ICD_DX_CD_2, - ICD_DX_CD_3::VARCHAR, - ICD_DX_CD_4::VARCHAR, - ICD_DX_CD_5::VARCHAR, - ICD_DX_CD_6::VARCHAR, - ICD_DX_CD_7::VARCHAR, - ICD_DX_CD_8::VARCHAR, - ICD_DX_CD_9::VARCHAR, - ICD_DX_CD_10::VARCHAR, - ICD_DX_CD_11::VARCHAR, - ICD_DX_CD_12::VARCHAR, - ICD_DX_CD_13::VARCHAR, - ICD_DX_CD_14::VARCHAR, - ICD_DX_CD_15::VARCHAR, - ICD_DX_CD_16::VARCHAR, - ICD_DX_CD_17::VARCHAR, - ICD_DX_CD_18::VARCHAR, - ICD_DX_CD_19::VARCHAR, - ICD_DX_CD_20::VARCHAR, - ICD_DX_CD_21::VARCHAR, - ICD_DX_CD_22::VARCHAR, - ICD_DX_CD_23::VARCHAR, - ICD_DX_CD_24::VARCHAR, + ICD_DX_CD_3::VARCHAR AS ICD_DX_CD_3, + ICD_DX_CD_4::VARCHAR AS ICD_DX_CD_4, + ICD_DX_CD_5::VARCHAR AS ICD_DX_CD_5, + ICD_DX_CD_6::VARCHAR AS ICD_DX_CD_6, + ICD_DX_CD_7::VARCHAR AS ICD_DX_CD_7, + ICD_DX_CD_8::VARCHAR AS ICD_DX_CD_8, + ICD_DX_CD_9::VARCHAR AS ICD_DX_CD_9, + ICD_DX_CD_10::VARCHAR AS ICD_DX_CD_10, + ICD_DX_CD_11::VARCHAR AS ICD_DX_CD_11, + ICD_DX_CD_12::VARCHAR AS ICD_DX_CD_12, + ICD_DX_CD_13::VARCHAR AS ICD_DX_CD_13, + ICD_DX_CD_14::VARCHAR AS ICD_DX_CD_14, + ICD_DX_CD_15::VARCHAR AS ICD_DX_CD_15, + ICD_DX_CD_16::VARCHAR AS ICD_DX_CD_16, + ICD_DX_CD_17::VARCHAR AS ICD_DX_CD_17, + ICD_DX_CD_18::VARCHAR AS ICD_DX_CD_18, + ICD_DX_CD_19::VARCHAR AS ICD_DX_CD_19, + ICD_DX_CD_20::VARCHAR AS ICD_DX_CD_20, + ICD_DX_CD_21::VARCHAR AS ICD_DX_CD_21, + ICD_DX_CD_22::VARCHAR AS ICD_DX_CD_22, + ICD_DX_CD_23::VARCHAR AS ICD_DX_CD_23, + ICD_DX_CD_24::VARCHAR AS ICD_DX_CD_24, ICD_DX_CD_25::VARCHAR AS ICD_DX_CD_25, ICD_PRCDR_CD_1::VARCHAR AS ICD_PRCDR_CD_1, ICD_PRCDR_CD_2::VARCHAR AS ICD_PRCDR_CD_2, - ICD_PRCDR_CD_3::VARCHAR, - ICD_PRCDR_CD_4::VARCHAR, - ICD_PRCDR_CD_5::VARCHAR, - ICD_PRCDR_CD_6::VARCHAR, - ICD_PRCDR_CD_7::VARCHAR, - ICD_PRCDR_CD_8::VARCHAR, - ICD_PRCDR_CD_9::VARCHAR, - ICD_PRCDR_CD_10::VARCHAR, - ICD_PRCDR_CD_11::VARCHAR, - ICD_PRCDR_CD_12::VARCHAR, - ICD_PRCDR_CD_13::VARCHAR, - ICD_PRCDR_CD_14::VARCHAR, - ICD_PRCDR_CD_15::VARCHAR, - ICD_PRCDR_CD_16::VARCHAR, - ICD_PRCDR_CD_17::VARCHAR, - ICD_PRCDR_CD_18::VARCHAR, - ICD_PRCDR_CD_19::VARCHAR, - ICD_PRCDR_CD_20::VARCHAR, - ICD_PRCDR_CD_21::VARCHAR, - ICD_PRCDR_CD_22::VARCHAR, - ICD_PRCDR_CD_23::VARCHAR, - ICD_PRCDR_CD_24::VARCHAR, + ICD_PRCDR_CD_3::VARCHAR AS ICD_PRCDR_CD_3, + ICD_PRCDR_CD_4::VARCHAR AS ICD_PRCDR_CD_4, + ICD_PRCDR_CD_5::VARCHAR AS ICD_PRCDR_CD_5, + ICD_PRCDR_CD_6::VARCHAR AS ICD_PRCDR_CD_6, + ICD_PRCDR_CD_7::VARCHAR AS ICD_PRCDR_CD_7, + ICD_PRCDR_CD_8::VARCHAR AS ICD_PRCDR_CD_8, + ICD_PRCDR_CD_9::VARCHAR AS ICD_PRCDR_CD_9, + ICD_PRCDR_CD_10::VARCHAR AS ICD_PRCDR_CD_10, + ICD_PRCDR_CD_11::VARCHAR AS ICD_PRCDR_CD_11, + ICD_PRCDR_CD_12::VARCHAR AS ICD_PRCDR_CD_12, + ICD_PRCDR_CD_13::VARCHAR AS ICD_PRCDR_CD_13, + ICD_PRCDR_CD_14::VARCHAR AS ICD_PRCDR_CD_14, + ICD_PRCDR_CD_15::VARCHAR AS ICD_PRCDR_CD_15, + ICD_PRCDR_CD_16::VARCHAR AS ICD_PRCDR_CD_16, + ICD_PRCDR_CD_17::VARCHAR AS ICD_PRCDR_CD_17, + ICD_PRCDR_CD_18::VARCHAR AS ICD_PRCDR_CD_18, + ICD_PRCDR_CD_19::VARCHAR AS ICD_PRCDR_CD_19, + ICD_PRCDR_CD_20::VARCHAR AS ICD_PRCDR_CD_20, + ICD_PRCDR_CD_21::VARCHAR AS ICD_PRCDR_CD_21, + ICD_PRCDR_CD_22::VARCHAR AS ICD_PRCDR_CD_22, + ICD_PRCDR_CD_23::VARCHAR AS ICD_PRCDR_CD_23, + ICD_PRCDR_CD_24::VARCHAR AS ICD_PRCDR_CD_24, ICD_PRCDR_CD_25::VARCHAR AS ICD_PRCDR_CD_25, CPT_PRCDR_CD_1::VARCHAR AS CPT_PRCDR_CD_1, - CPT_PRCDR_CD_2::VARCHAR, + CPT_PRCDR_CD_2::VARCHAR AS CPT_PRCDR_CD_2, CPT_PRCDR_CD_3::VARCHAR AS CPT_PRCDR_CD_3, - CPT_PRCDR_CD_4::VARCHAR, - CPT_PRCDR_CD_5::VARCHAR, - CPT_PRCDR_CD_6::VARCHAR, - CPT_PRCDR_CD_7::VARCHAR, - CPT_PRCDR_CD_8::VARCHAR, - CPT_PRCDR_CD_9::VARCHAR, - CPT_PRCDR_CD_10::VARCHAR, - CPT_PRCDR_CD_11::VARCHAR, - CPT_PRCDR_CD_12::VARCHAR, - CPT_PRCDR_CD_13::VARCHAR, - CPT_PRCDR_CD_14::VARCHAR, - CPT_PRCDR_CD_15::VARCHAR, - CPT_PRCDR_CD_16::VARCHAR, - CPT_PRCDR_CD_17::VARCHAR, - CPT_PRCDR_CD_18::VARCHAR, - CPT_PRCDR_CD_19::VARCHAR, - CPT_PRCDR_CD_20::VARCHAR, - CPT_PRCDR_CD_21::VARCHAR, - CPT_PRCDR_CD_22::VARCHAR, - CPT_PRCDR_CD_23::VARCHAR, - CPT_PRCDR_CD_24::VARCHAR, - CPT_PRCDR_CD_25::VARCHAR, - CPT_PRCDR_CD_26::VARCHAR, - CPT_PRCDR_CD_27::VARCHAR, - CPT_PRCDR_CD_28::VARCHAR, - CPT_PRCDR_CD_29::VARCHAR, - CPT_PRCDR_CD_30::VARCHAR, - CPT_PRCDR_CD_31::VARCHAR, - CPT_PRCDR_CD_32::VARCHAR, - CPT_PRCDR_CD_33::VARCHAR, - CPT_PRCDR_CD_34::VARCHAR, + CPT_PRCDR_CD_4::VARCHAR AS CPT_PRCDR_CD_4, + CPT_PRCDR_CD_5::VARCHAR AS CPT_PRCDR_CD_5, + CPT_PRCDR_CD_6::VARCHAR AS CPT_PRCDR_CD_6, + CPT_PRCDR_CD_7::VARCHAR AS CPT_PRCDR_CD_7, + CPT_PRCDR_CD_8::VARCHAR AS CPT_PRCDR_CD_8, + CPT_PRCDR_CD_9::VARCHAR AS CPT_PRCDR_CD_9, + CPT_PRCDR_CD_10::VARCHAR AS CPT_PRCDR_CD_10, + CPT_PRCDR_CD_11::VARCHAR AS CPT_PRCDR_CD_11, + CPT_PRCDR_CD_12::VARCHAR AS CPT_PRCDR_CD_12, + CPT_PRCDR_CD_13::VARCHAR AS CPT_PRCDR_CD_13, + CPT_PRCDR_CD_14::VARCHAR AS CPT_PRCDR_CD_14, + CPT_PRCDR_CD_15::VARCHAR AS CPT_PRCDR_CD_15, + CPT_PRCDR_CD_16::VARCHAR AS CPT_PRCDR_CD_16, + CPT_PRCDR_CD_17::VARCHAR AS CPT_PRCDR_CD_17, + CPT_PRCDR_CD_18::VARCHAR AS CPT_PRCDR_CD_18, + CPT_PRCDR_CD_19::VARCHAR AS CPT_PRCDR_CD_19, + CPT_PRCDR_CD_20::VARCHAR AS CPT_PRCDR_CD_20, + CPT_PRCDR_CD_21::VARCHAR AS CPT_PRCDR_CD_21, + CPT_PRCDR_CD_22::VARCHAR AS CPT_PRCDR_CD_22, + CPT_PRCDR_CD_23::VARCHAR AS CPT_PRCDR_CD_23, + CPT_PRCDR_CD_24::VARCHAR AS CPT_PRCDR_CD_24, + CPT_PRCDR_CD_25::VARCHAR AS CPT_PRCDR_CD_25, + CPT_PRCDR_CD_26::VARCHAR AS CPT_PRCDR_CD_26, + CPT_PRCDR_CD_27::VARCHAR AS CPT_PRCDR_CD_27, + CPT_PRCDR_CD_28::VARCHAR AS CPT_PRCDR_CD_28, + CPT_PRCDR_CD_29::VARCHAR AS CPT_PRCDR_CD_29, + CPT_PRCDR_CD_30::VARCHAR AS CPT_PRCDR_CD_30, + CPT_PRCDR_CD_31::VARCHAR AS CPT_PRCDR_CD_31, + CPT_PRCDR_CD_32::VARCHAR AS CPT_PRCDR_CD_32, + CPT_PRCDR_CD_33::VARCHAR AS CPT_PRCDR_CD_33, + CPT_PRCDR_CD_34::VARCHAR AS CPT_PRCDR_CD_34, CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicare_outpatient_2016.CSV', header=True, null_padding=true, types={'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR'}, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicare_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_person_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_person_2016.sql index 8e46a44..520ddc6 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_person_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_person_2016.sql @@ -6,58 +6,58 @@ SELECT PERSON_WGHT::NUMERIC AS PERSON_WGHT, AGE_LOW::NUMERIC AS AGE_LOW, AGE_HIGH::NUMERIC AS AGE_HIGH, - SEX_IDENT_CD::VARCHAR AS SEX_IDENT_CD, + SEX_IDENT_CD::UBIGINT AS SEX_IDENT_CD, RACE_CD::VARCHAR AS RACE_CD, STATE_CD::VARCHAR AS STATE_CD, COUNTY_FIPS_CD::VARCHAR AS COUNTY_FIPS_CD, ZIP_CD::VARCHAR AS ZIP_CD, RSN_ENRLMT_CD::VARCHAR AS RSN_ENRLMT_CD, MDCR_ENTLMT_IND_1::VARCHAR AS MDCR_ENTLMT_IND_1, - MDCR_ENTLMT_IND_2::VARCHAR, - MDCR_ENTLMT_IND_3::VARCHAR, - MDCR_ENTLMT_IND_4::VARCHAR, - MDCR_ENTLMT_IND_5::VARCHAR, - MDCR_ENTLMT_IND_6::VARCHAR, - MDCR_ENTLMT_IND_7::VARCHAR, - MDCR_ENTLMT_IND_8::VARCHAR, - MDCR_ENTLMT_IND_9::VARCHAR, - MDCR_ENTLMT_IND_10::VARCHAR, - MDCR_ENTLMT_IND_11::VARCHAR, + MDCR_ENTLMT_IND_2::VARCHAR AS MDCR_ENTLMT_IND_2, + MDCR_ENTLMT_IND_3::VARCHAR AS MDCR_ENTLMT_IND_3, + MDCR_ENTLMT_IND_4::VARCHAR AS MDCR_ENTLMT_IND_4, + MDCR_ENTLMT_IND_5::VARCHAR AS MDCR_ENTLMT_IND_5, + MDCR_ENTLMT_IND_6::VARCHAR AS MDCR_ENTLMT_IND_6, + MDCR_ENTLMT_IND_7::VARCHAR AS MDCR_ENTLMT_IND_7, + MDCR_ENTLMT_IND_8::VARCHAR AS MDCR_ENTLMT_IND_8, + MDCR_ENTLMT_IND_9::VARCHAR AS MDCR_ENTLMT_IND_9, + MDCR_ENTLMT_IND_10::VARCHAR AS MDCR_ENTLMT_IND_10, + MDCR_ENTLMT_IND_11::VARCHAR AS MDCR_ENTLMT_IND_11, MDCR_ENTLMT_IND_12::VARCHAR AS MDCR_ENTLMT_IND_12, MDCR_HMO_CVRG_1::NUMERIC AS MDCR_HMO_CVRG_1, - MDCR_HMO_CVRG_2::VARCHAR, - MDCR_HMO_CVRG_3::VARCHAR, - MDCR_HMO_CVRG_4::VARCHAR, - MDCR_HMO_CVRG_5::VARCHAR, - MDCR_HMO_CVRG_6::VARCHAR, - MDCR_HMO_CVRG_7::VARCHAR, - MDCR_HMO_CVRG_8::VARCHAR, - MDCR_HMO_CVRG_9::VARCHAR, - MDCR_HMO_CVRG_10::VARCHAR, - MDCR_HMO_CVRG_11::VARCHAR, + MDCR_HMO_CVRG_2::NUMERIC AS MDCR_HMO_CVRG_2, + MDCR_HMO_CVRG_3::NUMERIC AS MDCR_HMO_CVRG_3, + MDCR_HMO_CVRG_4::NUMERIC AS MDCR_HMO_CVRG_4, + MDCR_HMO_CVRG_5::NUMERIC AS MDCR_HMO_CVRG_5, + MDCR_HMO_CVRG_6::NUMERIC AS MDCR_HMO_CVRG_6, + MDCR_HMO_CVRG_7::NUMERIC AS MDCR_HMO_CVRG_7, + MDCR_HMO_CVRG_8::NUMERIC AS MDCR_HMO_CVRG_8, + MDCR_HMO_CVRG_9::NUMERIC AS MDCR_HMO_CVRG_9, + MDCR_HMO_CVRG_10::NUMERIC AS MDCR_HMO_CVRG_10, + MDCR_HMO_CVRG_11::NUMERIC AS MDCR_HMO_CVRG_11, MDCR_HMO_CVRG_12::NUMERIC AS MDCR_HMO_CVRG_12, PHRMCY_CVRG_1::NUMERIC AS PHRMCY_CVRG_1, - PHRMCY_CVRG_2::VARCHAR, - PHRMCY_CVRG_3::VARCHAR, - PHRMCY_CVRG_4::VARCHAR, - PHRMCY_CVRG_5::VARCHAR, - PHRMCY_CVRG_6::VARCHAR, - PHRMCY_CVRG_7::VARCHAR, - PHRMCY_CVRG_8::VARCHAR, - PHRMCY_CVRG_9::VARCHAR, - PHRMCY_CVRG_10::VARCHAR, - PHRMCY_CVRG_11::VARCHAR, + PHRMCY_CVRG_2::NUMERIC AS PHRMCY_CVRG_2, + PHRMCY_CVRG_3::NUMERIC AS PHRMCY_CVRG_3, + PHRMCY_CVRG_4::NUMERIC AS PHRMCY_CVRG_4, + PHRMCY_CVRG_5::NUMERIC AS PHRMCY_CVRG_5, + PHRMCY_CVRG_6::NUMERIC AS PHRMCY_CVRG_6, + PHRMCY_CVRG_7::NUMERIC AS PHRMCY_CVRG_7, + PHRMCY_CVRG_8::NUMERIC AS PHRMCY_CVRG_8, + PHRMCY_CVRG_9::NUMERIC AS PHRMCY_CVRG_9, + PHRMCY_CVRG_10::NUMERIC AS PHRMCY_CVRG_10, + PHRMCY_CVRG_11::NUMERIC AS PHRMCY_CVRG_11, PHRMCY_CVRG_12::NUMERIC AS PHRMCY_CVRG_12, DUAL_ELGBL_1::NUMERIC AS DUAL_ELGBL_1, - DUAL_ELGBL_2::VARCHAR, - DUAL_ELGBL_3::VARCHAR, - DUAL_ELGBL_4::VARCHAR, - DUAL_ELGBL_5::VARCHAR, - DUAL_ELGBL_6::VARCHAR, - DUAL_ELGBL_7::VARCHAR, - DUAL_ELGBL_8::VARCHAR, - DUAL_ELGBL_9::VARCHAR, - DUAL_ELGBL_10::VARCHAR, - DUAL_ELGBL_11::VARCHAR, + DUAL_ELGBL_2::NUMERIC AS DUAL_ELGBL_2, + DUAL_ELGBL_3::NUMERIC AS DUAL_ELGBL_3, + DUAL_ELGBL_4::NUMERIC AS DUAL_ELGBL_4, + DUAL_ELGBL_5::NUMERIC AS DUAL_ELGBL_5, + DUAL_ELGBL_6::NUMERIC AS DUAL_ELGBL_6, + DUAL_ELGBL_7::NUMERIC AS DUAL_ELGBL_7, + DUAL_ELGBL_8::NUMERIC AS DUAL_ELGBL_8, + DUAL_ELGBL_9::NUMERIC AS DUAL_ELGBL_9, + DUAL_ELGBL_10::NUMERIC AS DUAL_ELGBL_10, + DUAL_ELGBL_11::NUMERIC AS DUAL_ELGBL_11, DUAL_ELGBL_12::NUMERIC AS DUAL_ELGBL_12 -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicare_person_2016.CSV', header=True, null_padding=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicare_person_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'AGE_LOW': 'NUMERIC', 'AGE_HIGH': 'NUMERIC', 'SEX_IDENT_CD': 'UBIGINT', 'RACE_CD': 'VARCHAR', 'STATE_CD': 'VARCHAR', 'COUNTY_FIPS_CD': 'VARCHAR', 'ZIP_CD': 'VARCHAR', 'RSN_ENRLMT_CD': 'VARCHAR', 'MDCR_ENTLMT_IND_1': 'VARCHAR', 'MDCR_ENTLMT_IND_2': 'VARCHAR', 'MDCR_ENTLMT_IND_3': 'VARCHAR', 'MDCR_ENTLMT_IND_4': 'VARCHAR', 'MDCR_ENTLMT_IND_5': 'VARCHAR', 'MDCR_ENTLMT_IND_6': 'VARCHAR', 'MDCR_ENTLMT_IND_7': 'VARCHAR', 'MDCR_ENTLMT_IND_8': 'VARCHAR', 'MDCR_ENTLMT_IND_9': 'VARCHAR', 'MDCR_ENTLMT_IND_10': 'VARCHAR', 'MDCR_ENTLMT_IND_11': 'VARCHAR', 'MDCR_ENTLMT_IND_12': 'VARCHAR', 'MDCR_HMO_CVRG_1': 'NUMERIC', 'MDCR_HMO_CVRG_2': 'NUMERIC', 'MDCR_HMO_CVRG_3': 'NUMERIC', 'MDCR_HMO_CVRG_4': 'NUMERIC', 'MDCR_HMO_CVRG_5': 'NUMERIC', 'MDCR_HMO_CVRG_6': 'NUMERIC', 'MDCR_HMO_CVRG_7': 'NUMERIC', 'MDCR_HMO_CVRG_8': 'NUMERIC', 'MDCR_HMO_CVRG_9': 'NUMERIC', 'MDCR_HMO_CVRG_10': 'NUMERIC', 'MDCR_HMO_CVRG_11': 'NUMERIC', 'MDCR_HMO_CVRG_12': 'NUMERIC', 'PHRMCY_CVRG_1': 'NUMERIC', 'PHRMCY_CVRG_2': 'NUMERIC', 'PHRMCY_CVRG_3': 'NUMERIC', 'PHRMCY_CVRG_4': 'NUMERIC', 'PHRMCY_CVRG_5': 'NUMERIC', 'PHRMCY_CVRG_6': 'NUMERIC', 'PHRMCY_CVRG_7': 'NUMERIC', 'PHRMCY_CVRG_8': 'NUMERIC', 'PHRMCY_CVRG_9': 'NUMERIC', 'PHRMCY_CVRG_10': 'NUMERIC', 'PHRMCY_CVRG_11': 'NUMERIC', 'PHRMCY_CVRG_12': 'NUMERIC', 'DUAL_ELGBL_1': 'NUMERIC', 'DUAL_ELGBL_2': 'NUMERIC', 'DUAL_ELGBL_3': 'NUMERIC', 'DUAL_ELGBL_4': 'NUMERIC', 'DUAL_ELGBL_5': 'NUMERIC', 'DUAL_ELGBL_6': 'NUMERIC', 'DUAL_ELGBL_7': 'NUMERIC', 'DUAL_ELGBL_8': 'NUMERIC', 'DUAL_ELGBL_9': 'NUMERIC', 'DUAL_ELGBL_10': 'NUMERIC', 'DUAL_ELGBL_11': 'NUMERIC', 'DUAL_ELGBL_12': 'NUMERIC' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql index 53dc023..531580f 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql @@ -6,10 +6,10 @@ SELECT PERSON_WGHT::NUMERIC AS PERSON_WGHT, PHMCY_CLM_NUM::NUMERIC AS PHMCY_CLM_NUM, CLM_CNTL_NUM::NUMERIC AS CLM_CNTL_NUM, - LINE_NBR::VARCHAR, + LINE_NBR::VARCHAR AS LINE_NBR, FILL_DT::DATE AS FILL_DT, - SYNTHETIC_DRUG_ID::VARCHAR, + SYNTHETIC_DRUG_ID::VARCHAR AS SYNTHETIC_DRUG_ID, GENERIC_DRUG_NAME::VARCHAR AS GENERIC_DRUG_NAME, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicare_pharmacy_2016.CSV', header=True, null_padding=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicare_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_provider_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_provider_2016.sql index 80e0900..46276ce 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_provider_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_provider_2016.sql @@ -1,9 +1,9 @@ --- SQL model for syhdr_medicare_provider_2016.csv +-- SQL model for syhdr_medicare_provider_2016.CSV {{ config(materialized='external', location=var('output_path') + '/' + this.name + '.parquet') }} SELECT - Facility_ID::VARCHAR, - Prvdr_Ctgry_Cd::VARCHAR, - Prvdr_Ownrshp_Cd::VARCHAR, - Prvdr_Prtcptn_Cd::VARCHAR -FROM read_csv('/Users/me/data/syh_dr/syhdr_medicare_provider_2016.csv', header=True, null_padding=true) \ No newline at end of file + Facility_ID::VARCHAR AS Facility_ID, + Prvdr_Ctgry_Cd::VARCHAR AS Prvdr_Ctgry_Cd, + Prvdr_Ownrshp_Cd::VARCHAR AS Prvdr_Ownrshp_Cd, + Prvdr_Prtcptn_Cd::VARCHAR AS Prvdr_Prtcptn_Cd +FROM read_csv('~/data/syh_dr/syhdr_medicare_provider_2016.CSV', header=True, null_padding=true, types={ 'Facility_ID': 'VARCHAR', 'Prvdr_Ctgry_Cd': 'VARCHAR', 'Prvdr_Ownrshp_Cd': 'VARCHAR', 'Prvdr_Prtcptn_Cd': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file From 05aef1b12ee2ffde0670c40f2f3c43c8492a00b0 Mon Sep 17 00:00:00 2001 From: wesleycheung0 Date: Fri, 17 May 2024 14:53:38 -0400 Subject: [PATCH 3/5] Fixed: Not include Float in read_csv data type --- data_processing/scripts/generate_syh_dr_data_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data_processing/scripts/generate_syh_dr_data_models.py b/data_processing/scripts/generate_syh_dr_data_models.py index 5ad2a65..472f812 100644 --- a/data_processing/scripts/generate_syh_dr_data_models.py +++ b/data_processing/scripts/generate_syh_dr_data_models.py @@ -105,7 +105,8 @@ def process_csv_files(data_dictionary_df, csv_folder): for column_name in column_names: if column_name in column_definitions: data_type = column_definitions[column_name] - types_list.append(f"'{column_name}': '{data_type}'") + if data_type != "FLOAT": + types_list.append(f"'{column_name}': '{data_type}'") if "AMT" in column_name: # remove dollar sign from the VARCHAR string column_list.append( From 18f9545ae744f7e577cf47e2d6ec81721d201fd3 Mon Sep 17 00:00:00 2001 From: wesleycheung0 Date: Fri, 17 May 2024 14:54:04 -0400 Subject: [PATCH 4/5] Fixed: Not include Float in read_csv data type --- .../generated/with_types/syhdr_commercial_inpatient_2016.sql | 2 +- .../generated/with_types/syhdr_commercial_outpatient_2016.sql | 2 +- .../generated/with_types/syhdr_commercial_pharmacy_2016.sql | 2 +- .../generated/with_types/syhdr_medicaid_inpatient_2016.sql | 2 +- .../generated/with_types/syhdr_medicaid_outpatient_2016.sql | 2 +- .../generated/with_types/syhdr_medicaid_pharmacy_2016.sql | 2 +- .../generated/with_types/syhdr_medicare_inpatient_2016.sql | 2 +- .../generated/with_types/syhdr_medicare_outpatient_2016.sql | 2 +- .../generated/with_types/syhdr_medicare_pharmacy_2016.sql | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql index 1c82356..7da99fb 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_inpatient_2016.sql @@ -103,4 +103,4 @@ SELECT CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_commercial_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_commercial_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql index fa79128..9807157 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_outpatient_2016.sql @@ -103,4 +103,4 @@ SELECT CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_commercial_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_commercial_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql index e706ff0..d73279a 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_commercial_pharmacy_2016.sql @@ -12,4 +12,4 @@ SELECT GENERIC_DRUG_NAME::VARCHAR AS GENERIC_DRUG_NAME, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_commercial_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_commercial_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql index 5b4594d..3f67926 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_inpatient_2016.sql @@ -103,4 +103,4 @@ SELECT CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_medicaid_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicaid_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql index 95f8178..7268ad2 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_outpatient_2016.sql @@ -103,4 +103,4 @@ SELECT CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_medicaid_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicaid_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql index d2cff95..90165ef 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicaid_pharmacy_2016.sql @@ -12,4 +12,4 @@ SELECT GENERIC_DRUG_NAME::VARCHAR AS GENERIC_DRUG_NAME, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_medicaid_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicaid_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql index fd2ad3f..0b0cbac 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_inpatient_2016.sql @@ -103,4 +103,4 @@ SELECT CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_medicare_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicare_inpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql index 77f58f0..09f4e40 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_outpatient_2016.sql @@ -103,4 +103,4 @@ SELECT CPT_PRCDR_CD_35::VARCHAR AS CPT_PRCDR_CD_35, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_medicare_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicare_outpatient_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'FACILITY_ID': 'UBIGINT', 'CLM_CNTL_NUM': 'NUMERIC', 'AT_SPCLTY': 'VARCHAR', 'SRVC_BEG_DATE': 'DATE', 'SRVC_END_DATE': 'DATE', 'LOS': 'UINTEGER', 'ADMSN_TYPE': 'VARCHAR', 'TOB_CD': 'VARCHAR', 'CLM_TYPE_CD': 'VARCHAR', 'DSCHRG_STUS': 'VARCHAR', 'PRMRY_DX_IMPUTED': 'NUMERIC', 'PRMRY_DX_CD': 'VARCHAR', 'ICD_DX_CD_1': 'VARCHAR', 'ICD_DX_CD_2': 'VARCHAR', 'ICD_DX_CD_3': 'VARCHAR', 'ICD_DX_CD_4': 'VARCHAR', 'ICD_DX_CD_5': 'VARCHAR', 'ICD_DX_CD_6': 'VARCHAR', 'ICD_DX_CD_7': 'VARCHAR', 'ICD_DX_CD_8': 'VARCHAR', 'ICD_DX_CD_9': 'VARCHAR', 'ICD_DX_CD_10': 'VARCHAR', 'ICD_DX_CD_11': 'VARCHAR', 'ICD_DX_CD_12': 'VARCHAR', 'ICD_DX_CD_13': 'VARCHAR', 'ICD_DX_CD_14': 'VARCHAR', 'ICD_DX_CD_15': 'VARCHAR', 'ICD_DX_CD_16': 'VARCHAR', 'ICD_DX_CD_17': 'VARCHAR', 'ICD_DX_CD_18': 'VARCHAR', 'ICD_DX_CD_19': 'VARCHAR', 'ICD_DX_CD_20': 'VARCHAR', 'ICD_DX_CD_21': 'VARCHAR', 'ICD_DX_CD_22': 'VARCHAR', 'ICD_DX_CD_23': 'VARCHAR', 'ICD_DX_CD_24': 'VARCHAR', 'ICD_DX_CD_25': 'VARCHAR', 'ICD_PRCDR_CD_1': 'VARCHAR', 'ICD_PRCDR_CD_2': 'VARCHAR', 'ICD_PRCDR_CD_3': 'VARCHAR', 'ICD_PRCDR_CD_4': 'VARCHAR', 'ICD_PRCDR_CD_5': 'VARCHAR', 'ICD_PRCDR_CD_6': 'VARCHAR', 'ICD_PRCDR_CD_7': 'VARCHAR', 'ICD_PRCDR_CD_8': 'VARCHAR', 'ICD_PRCDR_CD_9': 'VARCHAR', 'ICD_PRCDR_CD_10': 'VARCHAR', 'ICD_PRCDR_CD_11': 'VARCHAR', 'ICD_PRCDR_CD_12': 'VARCHAR', 'ICD_PRCDR_CD_13': 'VARCHAR', 'ICD_PRCDR_CD_14': 'VARCHAR', 'ICD_PRCDR_CD_15': 'VARCHAR', 'ICD_PRCDR_CD_16': 'VARCHAR', 'ICD_PRCDR_CD_17': 'VARCHAR', 'ICD_PRCDR_CD_18': 'VARCHAR', 'ICD_PRCDR_CD_19': 'VARCHAR', 'ICD_PRCDR_CD_20': 'VARCHAR', 'ICD_PRCDR_CD_21': 'VARCHAR', 'ICD_PRCDR_CD_22': 'VARCHAR', 'ICD_PRCDR_CD_23': 'VARCHAR', 'ICD_PRCDR_CD_24': 'VARCHAR', 'ICD_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_1': 'VARCHAR', 'CPT_PRCDR_CD_2': 'VARCHAR', 'CPT_PRCDR_CD_3': 'VARCHAR', 'CPT_PRCDR_CD_4': 'VARCHAR', 'CPT_PRCDR_CD_5': 'VARCHAR', 'CPT_PRCDR_CD_6': 'VARCHAR', 'CPT_PRCDR_CD_7': 'VARCHAR', 'CPT_PRCDR_CD_8': 'VARCHAR', 'CPT_PRCDR_CD_9': 'VARCHAR', 'CPT_PRCDR_CD_10': 'VARCHAR', 'CPT_PRCDR_CD_11': 'VARCHAR', 'CPT_PRCDR_CD_12': 'VARCHAR', 'CPT_PRCDR_CD_13': 'VARCHAR', 'CPT_PRCDR_CD_14': 'VARCHAR', 'CPT_PRCDR_CD_15': 'VARCHAR', 'CPT_PRCDR_CD_16': 'VARCHAR', 'CPT_PRCDR_CD_17': 'VARCHAR', 'CPT_PRCDR_CD_18': 'VARCHAR', 'CPT_PRCDR_CD_19': 'VARCHAR', 'CPT_PRCDR_CD_20': 'VARCHAR', 'CPT_PRCDR_CD_21': 'VARCHAR', 'CPT_PRCDR_CD_22': 'VARCHAR', 'CPT_PRCDR_CD_23': 'VARCHAR', 'CPT_PRCDR_CD_24': 'VARCHAR', 'CPT_PRCDR_CD_25': 'VARCHAR', 'CPT_PRCDR_CD_26': 'VARCHAR', 'CPT_PRCDR_CD_27': 'VARCHAR', 'CPT_PRCDR_CD_28': 'VARCHAR', 'CPT_PRCDR_CD_29': 'VARCHAR', 'CPT_PRCDR_CD_30': 'VARCHAR', 'CPT_PRCDR_CD_31': 'VARCHAR', 'CPT_PRCDR_CD_32': 'VARCHAR', 'CPT_PRCDR_CD_33': 'VARCHAR', 'CPT_PRCDR_CD_34': 'VARCHAR', 'CPT_PRCDR_CD_35': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file diff --git a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql index 531580f..eefffc6 100644 --- a/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql +++ b/data_processing/models/ahrq.gov/generated/with_types/syhdr_medicare_pharmacy_2016.sql @@ -12,4 +12,4 @@ SELECT GENERIC_DRUG_NAME::VARCHAR AS GENERIC_DRUG_NAME, replace(replace(PLAN_PMT_AMT, '$', ''), ',', '')::FLOAT AS PLAN_PMT_AMT, replace(replace(TOT_CHRG_AMT, '$', ''), ',', '')::FLOAT AS TOT_CHRG_AMT -FROM read_csv('~/data/syh_dr/syhdr_medicare_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR', 'PLAN_PMT_AMT': 'FLOAT', 'TOT_CHRG_AMT': 'FLOAT' }, ignore_errors=true) \ No newline at end of file +FROM read_csv('~/data/syh_dr/syhdr_medicare_pharmacy_2016.CSV', header=True, null_padding=true, types={ 'PERSON_ID': 'UBIGINT', 'PERSON_WGHT': 'NUMERIC', 'PHMCY_CLM_NUM': 'NUMERIC', 'CLM_CNTL_NUM': 'NUMERIC', 'LINE_NBR': 'VARCHAR', 'FILL_DT': 'DATE', 'SYNTHETIC_DRUG_ID': 'VARCHAR', 'GENERIC_DRUG_NAME': 'VARCHAR' }, ignore_errors=true) \ No newline at end of file From 5b308c6d09c6f6346e7b297d3084c40110e2457e Mon Sep 17 00:00:00 2001 From: wesleycheung0 Date: Sat, 1 Jun 2024 03:21:33 -0400 Subject: [PATCH 5/5] Changed data dictionary sources --- .../scripts/generate_syh_dr_data_models.py | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/data_processing/scripts/generate_syh_dr_data_models.py b/data_processing/scripts/generate_syh_dr_data_models.py index b267510..6b776c6 100644 --- a/data_processing/scripts/generate_syh_dr_data_models.py +++ b/data_processing/scripts/generate_syh_dr_data_models.py @@ -18,23 +18,35 @@ def create_data_dictionary(): AHRQ_BASE_URL = 'https://www.ahrq.gov' SYH_DR_URL = 'https://www.ahrq.gov/data/innovations/syh-dr.html' - # Get all links to Data Dictionary CSV files + # Get the link to the Data Dictionary XLSX file response = requests.get(SYH_DR_URL) soup = BeautifulSoup(response.text, 'html.parser') - # Create list of data dictionary URLs (i.e., where attribute is like "Variables") - data_dictionary_urls = [] - for url in soup.find_all("a", string=re.compile("Variables")): - csv_url = AHRQ_BASE_URL + url.attrs.get("href") - data_dictionary_urls.append(csv_url) + # Extract the URL of the data dictionary XLSX file + xlsx_url = None + for a_tag in soup.find_all("a", href=True): + if re.search(r"\.xlsx$", a_tag['href']): + xlsx_url = AHRQ_BASE_URL + a_tag['href'] + break - # Concat all data dictionaries into a dataframe and write to a file + if not xlsx_url: + raise Exception("Data Dictionary XLSX file URL not found.") + + # Read the specified worksheets into dataframes ssl._create_default_https_context = ssl._create_unverified_context - data_dictionary_df = pd.concat((pd.read_csv(url) for url in data_dictionary_urls), ignore_index=True) - data_dictionary_df = data_dictionary_df[data_dictionary_df["Variable Name"].notnull()] - data_dictionary_df.to_csv("./data_processing/seeds/SyH-DR_data_dictionary.csv", index=False) + xls = pd.ExcelFile(xlsx_url) + + # Specify the worksheets to concatenate + worktabs_to_concat = ["Person-Level Variables", "Claims-Level Variables", "Pharmacy Variables"] + dfs = [xls.parse(sheet_name) for sheet_name in worktabs_to_concat] + + # Concatenate the dataframes + data_dictionary_df = pd.concat(dfs, ignore_index=True) + + # Save the concatenated dataframe to an xlsx file + data_dictionary_df.to_excel("./data_processing/seeds/SyH-DR_data_dictionary_combined.xlsx", index=False) - return data_dictionary_df[["Variable Label", "Variable Name", "Variable Type", "Variable Format", "Variable Length"]] + return data_dictionary_df def map_variable_type_to_sql(variable_name, variable_type, variable_length, variable_format): if "_ID" in variable_name: