added ocean variable list

[larsbuntemeyer]

fixed issues with the ocean variables according to #33 using

import pandas as pd

def retrieve_cmip6_mip_tables():
    """retrieve and concat all cmip6 mip tables from
    https://c6dreq.dkrz.de/docs/CMIP6_MIP_tables.xlsx
    """
    cols = [
        "frequency",
        "modeling_realm",
        "standard_name",
        "units",
        "cell_methods",
        "cell_measures",
        "long_name",
        "comment",
        "dimensions",
        "out_name",
        "type",
        "positive",
        "valid_min",
        "valid_max",
        "ok_min_mean_abs",
        "ok_max_mean_abs",
        "cmip6_table",
    ]
    cmip6_mip_tables_url = "https://c6dreq.dkrz.de/docs/CMIP6_MIP_tables.xlsx"
    tables = pd.read_excel(cmip6_mip_tables_url, sheet_name=None)
    del tables["Notes"]

    def add_table_name(df, table):
        df["cmip6_table"] = table
        return df

    df = pd.concat(add_table_name(df, table) for table, df in tables.items())
    df.rename(
        columns={
            "CF Standard Name": "standard_name",
            "Long name": "long_name",
            "Variable Name": "out_name",
        },
        inplace=True,
    )
    return df[cols].drop_duplicates(ignore_index=True)

df = pd.read_csv("cmor-table/datasets.csv")

ocean = pd.read_csv("ocean-variables.csv")
cmip6 = retrieve_cmip6_mip_tables()
cmip6["frequency"] = cmip6.frequency.str.rstrip("Pt")
index_cols = ["out_name", "standard_name", "frequency", "long_name"]

def choose_units(row):
    if ~pd.isnull(row.units_cmip6):
        #print(type(row.units_cmip6))
        return row.units_cmip6
    return "Hello"

# join with cmip6
merge = pd.merge(ocean, cmip6, on=index_cols, how="left", suffixes=("_med", "_cmip6"))#[df.columns]

# choose units
merge["units"] = merge.units_cmip6
merge["units"] = merge.units.where(~merge.units.isnull(), merge.units_med)

# replace invalid standard names
rename_dict = {
    'sea_water_z_velocity': 'upward_sea_water_velocity',
    'surface_x_velocity': 'surface_eastward_sea_water_velocity',
    'surface_y_velocity': 'surface_northward_sea_water_velocity',
}
merge.replace(rename_dict, inplace=True)

# now update tauuo and tauvo long_name, standard_name and units
merge.loc[merge.out_name == "tauuo", ["long_name", "standard_name", "units", "cell_methods"]] = cmip6.set_index(["out_name", "frequency"]).loc[("tauuo", "dec")][["long_name", "standard_name", "units", "cell_methods"]].values[0]
merge.loc[merge.out_name == "tauvo", ["long_name", "standard_name", "units", "cell_methods"]] = cmip6.set_index(["out_name", "frequency"]).loc[("tauvo", "dec")][["long_name", "standard_name", "units", "cell_methods"]].values[0]

# drop any rows with gibraltar in the standard_name
merge = merge[~merge.standard_name.str.contains("gibraltar")]

# use cmip6 comment for cmor table
merge.rename(columns={"comment_cmip6": "comment"}, inplace=True)

for col in df.columns:
    if col not in merge.columns:
        merge.rename(columns={col + "_cmip6": col}, inplace=True)

# upate cmor tables
new_df = pd.concat([df, merge[df.columns]], ignore_index=True)#.to_csv("cmor-table/datasets.csv", index=False)

# create data request table
dreq_cols = ["out_name", "frequency", "units", "long_name", "standard_name", "cell_methods", "priority", "comment_med"]
dreq_df = merge[dreq_cols].rename(columns={"comment_med": "comment"})