From 5367f3ad2b61858bb1e0abd1af8e92ad888953b0 Mon Sep 17 00:00:00 2001 From: sabinala Date: Mon, 15 Jan 2024 10:45:24 -0800 Subject: [PATCH 1/7] adding testing check_data function to load_data --- docs/source/test_data_errors.ipynb | 168 ++++++++++++++++++++++ pyciemss/integration_utils/observation.py | 18 ++- 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 docs/source/test_data_errors.ipynb diff --git a/docs/source/test_data_errors.ipynb b/docs/source/test_data_errors.ipynb new file mode 100644 index 000000000..6ff9f8a20 --- /dev/null +++ b/docs/source/test_data_errors.ipynb @@ -0,0 +1,168 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "715031d6", + "metadata": {}, + "source": [ + "## This notebook tests different types of possible data errors, and makes sure that they are handled gracefully within the `load_data` function." + ] + }, + { + "cell_type": "markdown", + "id": "e6566b20", + "metadata": {}, + "source": [ + "### Load dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "75ce8777", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pyciemss\n", + "from pyciemss.integration_utils.observation import load_data" + ] + }, + { + "cell_type": "markdown", + "id": "1cbcff86", + "metadata": {}, + "source": [ + "### Set the data path and data mapping" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f6447da", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_PATH = \"https://raw.githubusercontent.com/ciemss/pyciemss/sa-load-data-tests/docs/source/sa-testing-notebooks/\"\n", + "data_mapping = {\"case\": \"I\", \"hosp\": \"H\"}" + ] + }, + { + "cell_type": "markdown", + "id": "b641d510", + "metadata": {}, + "source": [ + "### Check for errors with different data sets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b70cf51b", + "metadata": {}, + "outputs": [], + "source": [ + "# There should be no errors here\n", + "dataset = os.path.join(DATA_PATH, \"SIR_data_case_hosp.csv\")\n", + "load_data(dataset, data_mapping) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "840ce831", + "metadata": {}, + "outputs": [], + "source": [ + "# Should throw an error for missing data\n", + "dataset = os.path.join(DATA_PATH, \"SIR_missing_data.csv\")\n", + "load_data(dataset, data_mapping)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8d07064", + "metadata": {}, + "outputs": [], + "source": [ + "# Should throw an error for NaN-type data\n", + "dataset = os.path.join(DATA_PATH, \"SIR_data_nan.csv\")\n", + "load_data(dataset, data_mapping)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26c24d2f", + "metadata": {}, + "outputs": [], + "source": [ + "# Should throw an error for missing data in the form of a space\n", + "dataset = os.path.join(DATA_PATH, \"SIR_missing_data_space.csv\")\n", + "load_data(dataset, data_mapping)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "583d7b4a", + "metadata": {}, + "outputs": [], + "source": [ + "# Should throw an error for None-type data\n", + "dataset = os.path.join(DATA_PATH, \"SIR_data_none_type.csv\")\n", + "load_data(dataset, data_mapping)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d417aab", + "metadata": {}, + "outputs": [], + "source": [ + "# Should throw an error for having the wrong column name\n", + "dataset = os.path.join(DATA_PATH, \"SIR_data_wrong_time_col_name.csv\")\n", + "load_data(dataset, data_mapping)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64badecc", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "298ee978", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (pyciemss)", + "language": "python", + "name": "pyciemss" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyciemss/integration_utils/observation.py b/pyciemss/integration_utils/observation.py index 7e24b6d37..a333b4d10 100644 --- a/pyciemss/integration_utils/observation.py +++ b/pyciemss/integration_utils/observation.py @@ -21,7 +21,23 @@ def load_data( - If not provided, we will assume that the column names in the data file match the state variable names. """ - df = pd.read_csv(path) + def check_data(data_path: str): + # This function checks a dataset formatting errors, and returns a dataframe + data_df = pd.read_csv(path) + print(data_df.head()) + + # Check that the first column name is "Timestamp" + assert data_df.columns[0] == "Timestamp", "The column of timepoints must be first, and named 'Timestamp'." + + # Check that there are no NaN values or empty entries + assert not data_df.isna().any().any(), "Dataset cannot contain NaN or empty entries" + + # Check that there is no missing data in the form of None type or char values + assert data_df.applymap(lambda x: isinstance(x, (int, float))).all().all(), "Dataset cannot contain None type or char values. All entries must be of type `int` or `float`." + + return data_df + + df = check_data(path) data_timepoints = torch.tensor(df["Timestamp"].values, dtype=torch.float32) data = {} From f888221b0f5847893ec6b08a9bebce416616a223 Mon Sep 17 00:00:00 2001 From: sabinala Date: Mon, 15 Jan 2024 10:55:25 -0800 Subject: [PATCH 2/7] reformatting --- pyciemss/integration_utils/observation.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pyciemss/integration_utils/observation.py b/pyciemss/integration_utils/observation.py index a333b4d10..6736119e0 100644 --- a/pyciemss/integration_utils/observation.py +++ b/pyciemss/integration_utils/observation.py @@ -25,18 +25,24 @@ def check_data(data_path: str): # This function checks a dataset formatting errors, and returns a dataframe data_df = pd.read_csv(path) print(data_df.head()) - + # Check that the first column name is "Timestamp" - assert data_df.columns[0] == "Timestamp", "The column of timepoints must be first, and named 'Timestamp'." - + assert ( + data_df.columns[0] == "Timestamp" + ), "The column of timepoints must be first, and named 'Timestamp'." + # Check that there are no NaN values or empty entries - assert not data_df.isna().any().any(), "Dataset cannot contain NaN or empty entries" - + assert ( + not data_df.isna().any().any() + ), "Dataset cannot contain NaN or empty entries" + # Check that there is no missing data in the form of None type or char values - assert data_df.applymap(lambda x: isinstance(x, (int, float))).all().all(), "Dataset cannot contain None type or char values. All entries must be of type `int` or `float`." - + assert ( + data_df.applymap(lambda x: isinstance(x, (int, float))).all().all() + ), "Dataset cannot contain None type or char values. All entries must be of type `int` or `float`." + return data_df - + df = check_data(path) data_timepoints = torch.tensor(df["Timestamp"].values, dtype=torch.float32) From f5024acbbfe9d82f35c7739c74c8b48637f702ea Mon Sep 17 00:00:00 2001 From: sabinala Date: Tue, 13 Feb 2024 12:21:45 -0800 Subject: [PATCH 3/7] fixing data checker --- docs/source/test_data_errors.ipynb | 48 ++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/source/test_data_errors.ipynb b/docs/source/test_data_errors.ipynb index 6ff9f8a20..185c013cd 100644 --- a/docs/source/test_data_errors.ipynb +++ b/docs/source/test_data_errors.ipynb @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "8f6447da", "metadata": {}, "outputs": [], @@ -57,10 +57,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "b70cf51b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "URLError", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mSSLCertVerificationError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:1348\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m 1347\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1348\u001b[0m \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mselector\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhas_header\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mTransfer-encoding\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1282\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1281\u001b[0m \u001b[38;5;124;03m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1282\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1328\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1327\u001b[0m body \u001b[38;5;241m=\u001b[39m _encode(body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbody\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m-> 1328\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1277\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1276\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1277\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1037\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1036\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1037\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1039\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1040\u001b[0m \n\u001b[1;32m 1041\u001b[0m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:975\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[0;32m--> 975\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 976\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1454\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1452\u001b[0m server_hostname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhost\n\u001b[0;32m-> 1454\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrap_socket\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1455\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:513\u001b[0m, in \u001b[0;36mSSLContext.wrap_socket\u001b[0;34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001b[0m\n\u001b[1;32m 507\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrap_socket\u001b[39m(\u001b[38;5;28mself\u001b[39m, sock, server_side\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 508\u001b[0m do_handshake_on_connect\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 509\u001b[0m suppress_ragged_eofs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 510\u001b[0m server_hostname\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, session\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 511\u001b[0m \u001b[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001b[39;00m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;66;03m# ctx._wrap_socket()\u001b[39;00m\n\u001b[0;32m--> 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msslsocket_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 514\u001b[0m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 515\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_side\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_side\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 516\u001b[0m \u001b[43m \u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 517\u001b[0m \u001b[43m \u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 518\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 519\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 520\u001b[0m \u001b[43m \u001b[49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msession\u001b[49m\n\u001b[1;32m 521\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:1071\u001b[0m, in \u001b[0;36mSSLSocket._create\u001b[0;34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001b[0m\n\u001b[1;32m 1070\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdo_handshake_on_connect should not be specified for non-blocking sockets\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1071\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1072\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mOSError\u001b[39;00m, \u001b[38;5;167;01mValueError\u001b[39;00m):\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:1342\u001b[0m, in \u001b[0;36mSSLSocket.do_handshake\u001b[0;34m(self, block)\u001b[0m\n\u001b[1;32m 1341\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettimeout(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1342\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1343\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n", + "\u001b[0;31mSSLCertVerificationError\u001b[0m: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:997)", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mURLError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[3], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# There should be no errors here\u001b[39;00m\n\u001b[1;32m 2\u001b[0m dataset \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(DATA_PATH, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSIR_data_case_hosp.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[43mload_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata_mapping\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Projects/pyciemss/pyciemss/integration_utils/observation.py:46\u001b[0m, in \u001b[0;36mload_data\u001b[0;34m(path, data_mapping)\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m (\n\u001b[1;32m 41\u001b[0m data_df\u001b[38;5;241m.\u001b[39mapplymap(\u001b[38;5;28;01mlambda\u001b[39;00m x: \u001b[38;5;28misinstance\u001b[39m(x, (\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mfloat\u001b[39m)))\u001b[38;5;241m.\u001b[39mall()\u001b[38;5;241m.\u001b[39mall()\n\u001b[1;32m 42\u001b[0m ), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDataset cannot contain None type or char values. All entries must be of type `int` or `float`.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m data_df\n\u001b[0;32m---> 46\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mcheck_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 48\u001b[0m data_timepoints \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor(df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTimestamp\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalues, dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mfloat32)\n\u001b[1;32m 49\u001b[0m data \u001b[38;5;241m=\u001b[39m {}\n", + "File \u001b[0;32m~/Projects/pyciemss/pyciemss/integration_utils/observation.py:26\u001b[0m, in \u001b[0;36mload_data..check_data\u001b[0;34m(data_path)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcheck_data\u001b[39m(data_path: \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m 25\u001b[0m \u001b[38;5;66;03m# This function checks a dataset formatting errors, and returns a dataframe\u001b[39;00m\n\u001b[0;32m---> 26\u001b[0m data_df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(data_df\u001b[38;5;241m.\u001b[39mhead())\n\u001b[1;32m 29\u001b[0m \u001b[38;5;66;03m# Check that the first column name is \"Timestamp\"\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/util/_decorators.py:211\u001b[0m, in \u001b[0;36mdeprecate_kwarg.._deprecate_kwarg..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 210\u001b[0m kwargs[new_arg_name] \u001b[38;5;241m=\u001b[39m new_arg_value\n\u001b[0;32m--> 211\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/util/_decorators.py:331\u001b[0m, in \u001b[0;36mdeprecate_nonkeyword_arguments..decorate..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m>\u001b[39m num_allow_args:\n\u001b[1;32m 326\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m 327\u001b[0m msg\u001b[38;5;241m.\u001b[39mformat(arguments\u001b[38;5;241m=\u001b[39m_format_argument_list(allow_args)),\n\u001b[1;32m 328\u001b[0m \u001b[38;5;167;01mFutureWarning\u001b[39;00m,\n\u001b[1;32m 329\u001b[0m stacklevel\u001b[38;5;241m=\u001b[39mfind_stack_level(),\n\u001b[1;32m 330\u001b[0m )\n\u001b[0;32m--> 331\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:950\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m 935\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m 936\u001b[0m dialect,\n\u001b[1;32m 937\u001b[0m delimiter,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 946\u001b[0m defaults\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdelimiter\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m},\n\u001b[1;32m 947\u001b[0m )\n\u001b[1;32m 948\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m--> 950\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:605\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 602\u001b[0m _validate_names(kwds\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnames\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[1;32m 604\u001b[0m \u001b[38;5;66;03m# Create the parser.\u001b[39;00m\n\u001b[0;32m--> 605\u001b[0m parser \u001b[38;5;241m=\u001b[39m \u001b[43mTextFileReader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 607\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunksize \u001b[38;5;129;01mor\u001b[39;00m iterator:\n\u001b[1;32m 608\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parser\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1442\u001b[0m, in \u001b[0;36mTextFileReader.__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 1439\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptions[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m kwds[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 1441\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles: IOHandles \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 1442\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1735\u001b[0m, in \u001b[0;36mTextFileReader._make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m 1733\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m mode:\n\u001b[1;32m 1734\u001b[0m mode \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m-> 1735\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;241m=\u001b[39m \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1736\u001b[0m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1737\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1738\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1739\u001b[0m \u001b[43m \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcompression\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1740\u001b[0m \u001b[43m \u001b[49m\u001b[43mmemory_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmemory_map\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1741\u001b[0m \u001b[43m \u001b[49m\u001b[43mis_text\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_text\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1742\u001b[0m \u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding_errors\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstrict\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1743\u001b[0m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstorage_options\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1744\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1745\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 1746\u001b[0m f \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles\u001b[38;5;241m.\u001b[39mhandle\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:713\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 710\u001b[0m codecs\u001b[38;5;241m.\u001b[39mlookup_error(errors)\n\u001b[1;32m 712\u001b[0m \u001b[38;5;66;03m# open URLs\u001b[39;00m\n\u001b[0;32m--> 713\u001b[0m ioargs \u001b[38;5;241m=\u001b[39m \u001b[43m_get_filepath_or_buffer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 714\u001b[0m \u001b[43m \u001b[49m\u001b[43mpath_or_buf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 715\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 716\u001b[0m \u001b[43m \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcompression\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 717\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 718\u001b[0m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 719\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 721\u001b[0m handle \u001b[38;5;241m=\u001b[39m ioargs\u001b[38;5;241m.\u001b[39mfilepath_or_buffer\n\u001b[1;32m 722\u001b[0m handles: \u001b[38;5;28mlist\u001b[39m[BaseBuffer]\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:363\u001b[0m, in \u001b[0;36m_get_filepath_or_buffer\u001b[0;34m(filepath_or_buffer, encoding, compression, mode, storage_options)\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[38;5;66;03m# assuming storage_options is to be interpreted as headers\u001b[39;00m\n\u001b[1;32m 362\u001b[0m req_info \u001b[38;5;241m=\u001b[39m urllib\u001b[38;5;241m.\u001b[39mrequest\u001b[38;5;241m.\u001b[39mRequest(filepath_or_buffer, headers\u001b[38;5;241m=\u001b[39mstorage_options)\n\u001b[0;32m--> 363\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq_info\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m req:\n\u001b[1;32m 364\u001b[0m content_encoding \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mContent-Encoding\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m 365\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m content_encoding \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgzip\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 366\u001b[0m \u001b[38;5;66;03m# Override compression based on Content-Encoding header\u001b[39;00m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:265\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 260\u001b[0m \u001b[38;5;124;03mLazy-import wrapper for stdlib urlopen, as that imports a big chunk of\u001b[39;00m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124;03mthe stdlib.\u001b[39;00m\n\u001b[1;32m 262\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 263\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01murllib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrequest\u001b[39;00m\n\u001b[0;32m--> 265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43murllib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:216\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 215\u001b[0m opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:519\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m 516\u001b[0m req \u001b[38;5;241m=\u001b[39m meth(req)\n\u001b[1;32m 518\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[0;32m--> 519\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 521\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n\u001b[1;32m 522\u001b[0m meth_name \u001b[38;5;241m=\u001b[39m protocol\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_response\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:536\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m 533\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m 535\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[0;32m--> 536\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[1;32m 537\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m 539\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:496\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m 494\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m 495\u001b[0m func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 496\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 498\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:1391\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m 1390\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[0;32m-> 1391\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1392\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:1351\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m 1348\u001b[0m h\u001b[38;5;241m.\u001b[39mrequest(req\u001b[38;5;241m.\u001b[39mget_method(), req\u001b[38;5;241m.\u001b[39mselector, req\u001b[38;5;241m.\u001b[39mdata, headers,\n\u001b[1;32m 1349\u001b[0m encode_chunked\u001b[38;5;241m=\u001b[39mreq\u001b[38;5;241m.\u001b[39mhas_header(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTransfer-encoding\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[1;32m 1350\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n\u001b[0;32m-> 1351\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m URLError(err)\n\u001b[1;32m 1352\u001b[0m r \u001b[38;5;241m=\u001b[39m h\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[1;32m 1353\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n", + "\u001b[0;31mURLError\u001b[0m: " + ] + } + ], "source": [ "# There should be no errors here\n", "dataset = os.path.join(DATA_PATH, \"SIR_data_case_hosp.csv\")\n", From 7b36abe8c839d2f602fa9f9247730a36f847fdbd Mon Sep 17 00:00:00 2001 From: sabinala Date: Mon, 19 Feb 2024 10:51:06 -0800 Subject: [PATCH 4/7] adding test for load_data --- docs/source/test_data_errors.ipynb | 210 ---------------------- pyciemss/integration_utils/observation.py | 53 ++++-- tests/fixtures.py | 35 ++++ tests/test_interfaces.py | 13 ++ 4 files changed, 86 insertions(+), 225 deletions(-) delete mode 100644 docs/source/test_data_errors.ipynb diff --git a/docs/source/test_data_errors.ipynb b/docs/source/test_data_errors.ipynb deleted file mode 100644 index 185c013cd..000000000 --- a/docs/source/test_data_errors.ipynb +++ /dev/null @@ -1,210 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "715031d6", - "metadata": {}, - "source": [ - "## This notebook tests different types of possible data errors, and makes sure that they are handled gracefully within the `load_data` function." - ] - }, - { - "cell_type": "markdown", - "id": "e6566b20", - "metadata": {}, - "source": [ - "### Load dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "75ce8777", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import pyciemss\n", - "from pyciemss.integration_utils.observation import load_data" - ] - }, - { - "cell_type": "markdown", - "id": "1cbcff86", - "metadata": {}, - "source": [ - "### Set the data path and data mapping" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "8f6447da", - "metadata": {}, - "outputs": [], - "source": [ - "DATA_PATH = \"https://raw.githubusercontent.com/ciemss/pyciemss/sa-load-data-tests/docs/source/sa-testing-notebooks/\"\n", - "data_mapping = {\"case\": \"I\", \"hosp\": \"H\"}" - ] - }, - { - "cell_type": "markdown", - "id": "b641d510", - "metadata": {}, - "source": [ - "### Check for errors with different data sets" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "b70cf51b", - "metadata": {}, - "outputs": [ - { - "ename": "URLError", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mSSLCertVerificationError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:1348\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m 1347\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1348\u001b[0m \u001b[43mh\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_method\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mselector\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1349\u001b[0m \u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhas_header\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mTransfer-encoding\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1350\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1282\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1281\u001b[0m \u001b[38;5;124;03m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1282\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_request\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1328\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1327\u001b[0m body \u001b[38;5;241m=\u001b[39m _encode(body, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbody\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m-> 1328\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1277\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1276\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1277\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1037\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1036\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1037\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1039\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 1040\u001b[0m \n\u001b[1;32m 1041\u001b[0m \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:975\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 974\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[0;32m--> 975\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 976\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/http/client.py:1454\u001b[0m, in \u001b[0;36mHTTPSConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1452\u001b[0m server_hostname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhost\n\u001b[0;32m-> 1454\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwrap_socket\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1455\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:513\u001b[0m, in \u001b[0;36mSSLContext.wrap_socket\u001b[0;34m(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)\u001b[0m\n\u001b[1;32m 507\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrap_socket\u001b[39m(\u001b[38;5;28mself\u001b[39m, sock, server_side\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m 508\u001b[0m do_handshake_on_connect\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 509\u001b[0m suppress_ragged_eofs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m 510\u001b[0m server_hostname\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, session\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 511\u001b[0m \u001b[38;5;66;03m# SSLSocket class handles server_hostname encoding before it calls\u001b[39;00m\n\u001b[1;32m 512\u001b[0m \u001b[38;5;66;03m# ctx._wrap_socket()\u001b[39;00m\n\u001b[0;32m--> 513\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msslsocket_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 514\u001b[0m \u001b[43m \u001b[49m\u001b[43msock\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msock\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 515\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_side\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_side\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 516\u001b[0m \u001b[43m \u001b[49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdo_handshake_on_connect\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 517\u001b[0m \u001b[43m \u001b[49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msuppress_ragged_eofs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 518\u001b[0m \u001b[43m \u001b[49m\u001b[43mserver_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mserver_hostname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 519\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m 520\u001b[0m \u001b[43m \u001b[49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msession\u001b[49m\n\u001b[1;32m 521\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:1071\u001b[0m, in \u001b[0;36mSSLSocket._create\u001b[0;34m(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)\u001b[0m\n\u001b[1;32m 1070\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdo_handshake_on_connect should not be specified for non-blocking sockets\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1071\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1072\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (\u001b[38;5;167;01mOSError\u001b[39;00m, \u001b[38;5;167;01mValueError\u001b[39;00m):\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/ssl.py:1342\u001b[0m, in \u001b[0;36mSSLSocket.do_handshake\u001b[0;34m(self, block)\u001b[0m\n\u001b[1;32m 1341\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettimeout(\u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 1342\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_handshake\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1343\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n", - "\u001b[0;31mSSLCertVerificationError\u001b[0m: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:997)", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mURLError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[3], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# There should be no errors here\u001b[39;00m\n\u001b[1;32m 2\u001b[0m dataset \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(DATA_PATH, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSIR_data_case_hosp.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m----> 3\u001b[0m \u001b[43mload_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata_mapping\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/Projects/pyciemss/pyciemss/integration_utils/observation.py:46\u001b[0m, in \u001b[0;36mload_data\u001b[0;34m(path, data_mapping)\u001b[0m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m (\n\u001b[1;32m 41\u001b[0m data_df\u001b[38;5;241m.\u001b[39mapplymap(\u001b[38;5;28;01mlambda\u001b[39;00m x: \u001b[38;5;28misinstance\u001b[39m(x, (\u001b[38;5;28mint\u001b[39m, \u001b[38;5;28mfloat\u001b[39m)))\u001b[38;5;241m.\u001b[39mall()\u001b[38;5;241m.\u001b[39mall()\n\u001b[1;32m 42\u001b[0m ), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDataset cannot contain None type or char values. All entries must be of type `int` or `float`.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m data_df\n\u001b[0;32m---> 46\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[43mcheck_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 48\u001b[0m data_timepoints \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor(df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTimestamp\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mvalues, dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mfloat32)\n\u001b[1;32m 49\u001b[0m data \u001b[38;5;241m=\u001b[39m {}\n", - "File \u001b[0;32m~/Projects/pyciemss/pyciemss/integration_utils/observation.py:26\u001b[0m, in \u001b[0;36mload_data..check_data\u001b[0;34m(data_path)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcheck_data\u001b[39m(data_path: \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m 25\u001b[0m \u001b[38;5;66;03m# This function checks a dataset formatting errors, and returns a dataframe\u001b[39;00m\n\u001b[0;32m---> 26\u001b[0m data_df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;28mprint\u001b[39m(data_df\u001b[38;5;241m.\u001b[39mhead())\n\u001b[1;32m 29\u001b[0m \u001b[38;5;66;03m# Check that the first column name is \"Timestamp\"\u001b[39;00m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/util/_decorators.py:211\u001b[0m, in \u001b[0;36mdeprecate_kwarg.._deprecate_kwarg..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 209\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 210\u001b[0m kwargs[new_arg_name] \u001b[38;5;241m=\u001b[39m new_arg_value\n\u001b[0;32m--> 211\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/util/_decorators.py:331\u001b[0m, in \u001b[0;36mdeprecate_nonkeyword_arguments..decorate..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 325\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m>\u001b[39m num_allow_args:\n\u001b[1;32m 326\u001b[0m warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m 327\u001b[0m msg\u001b[38;5;241m.\u001b[39mformat(arguments\u001b[38;5;241m=\u001b[39m_format_argument_list(allow_args)),\n\u001b[1;32m 328\u001b[0m \u001b[38;5;167;01mFutureWarning\u001b[39;00m,\n\u001b[1;32m 329\u001b[0m stacklevel\u001b[38;5;241m=\u001b[39mfind_stack_level(),\n\u001b[1;32m 330\u001b[0m )\n\u001b[0;32m--> 331\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:950\u001b[0m, in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m 935\u001b[0m kwds_defaults \u001b[38;5;241m=\u001b[39m _refine_defaults_read(\n\u001b[1;32m 936\u001b[0m dialect,\n\u001b[1;32m 937\u001b[0m delimiter,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 946\u001b[0m defaults\u001b[38;5;241m=\u001b[39m{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdelimiter\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m,\u001b[39m\u001b[38;5;124m\"\u001b[39m},\n\u001b[1;32m 947\u001b[0m )\n\u001b[1;32m 948\u001b[0m kwds\u001b[38;5;241m.\u001b[39mupdate(kwds_defaults)\n\u001b[0;32m--> 950\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_read\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:605\u001b[0m, in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 602\u001b[0m _validate_names(kwds\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnames\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[1;32m 604\u001b[0m \u001b[38;5;66;03m# Create the parser.\u001b[39;00m\n\u001b[0;32m--> 605\u001b[0m parser \u001b[38;5;241m=\u001b[39m \u001b[43mTextFileReader\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfilepath_or_buffer\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 607\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m chunksize \u001b[38;5;129;01mor\u001b[39;00m iterator:\n\u001b[1;32m 608\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m parser\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1442\u001b[0m, in \u001b[0;36mTextFileReader.__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 1439\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39moptions[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m kwds[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhas_index_names\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 1441\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles: IOHandles \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m-> 1442\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_engine \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mengine\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/parsers/readers.py:1735\u001b[0m, in \u001b[0;36mTextFileReader._make_engine\u001b[0;34m(self, f, engine)\u001b[0m\n\u001b[1;32m 1733\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m mode:\n\u001b[1;32m 1734\u001b[0m mode \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m-> 1735\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;241m=\u001b[39m \u001b[43mget_handle\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1736\u001b[0m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1737\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1738\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1739\u001b[0m \u001b[43m \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mcompression\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1740\u001b[0m \u001b[43m \u001b[49m\u001b[43mmemory_map\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mmemory_map\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1741\u001b[0m \u001b[43m \u001b[49m\u001b[43mis_text\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mis_text\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1742\u001b[0m \u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mencoding_errors\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstrict\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1743\u001b[0m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moptions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstorage_options\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1744\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1745\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 1746\u001b[0m f \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandles\u001b[38;5;241m.\u001b[39mhandle\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:713\u001b[0m, in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m 710\u001b[0m codecs\u001b[38;5;241m.\u001b[39mlookup_error(errors)\n\u001b[1;32m 712\u001b[0m \u001b[38;5;66;03m# open URLs\u001b[39;00m\n\u001b[0;32m--> 713\u001b[0m ioargs \u001b[38;5;241m=\u001b[39m \u001b[43m_get_filepath_or_buffer\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 714\u001b[0m \u001b[43m \u001b[49m\u001b[43mpath_or_buf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 715\u001b[0m \u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 716\u001b[0m \u001b[43m \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcompression\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 717\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 718\u001b[0m \u001b[43m \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 719\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 721\u001b[0m handle \u001b[38;5;241m=\u001b[39m ioargs\u001b[38;5;241m.\u001b[39mfilepath_or_buffer\n\u001b[1;32m 722\u001b[0m handles: \u001b[38;5;28mlist\u001b[39m[BaseBuffer]\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:363\u001b[0m, in \u001b[0;36m_get_filepath_or_buffer\u001b[0;34m(filepath_or_buffer, encoding, compression, mode, storage_options)\u001b[0m\n\u001b[1;32m 361\u001b[0m \u001b[38;5;66;03m# assuming storage_options is to be interpreted as headers\u001b[39;00m\n\u001b[1;32m 362\u001b[0m req_info \u001b[38;5;241m=\u001b[39m urllib\u001b[38;5;241m.\u001b[39mrequest\u001b[38;5;241m.\u001b[39mRequest(filepath_or_buffer, headers\u001b[38;5;241m=\u001b[39mstorage_options)\n\u001b[0;32m--> 363\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq_info\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m req:\n\u001b[1;32m 364\u001b[0m content_encoding \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mContent-Encoding\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m 365\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m content_encoding \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgzip\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 366\u001b[0m \u001b[38;5;66;03m# Override compression based on Content-Encoding header\u001b[39;00m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/site-packages/pandas/io/common.py:265\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 259\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 260\u001b[0m \u001b[38;5;124;03mLazy-import wrapper for stdlib urlopen, as that imports a big chunk of\u001b[39;00m\n\u001b[1;32m 261\u001b[0m \u001b[38;5;124;03mthe stdlib.\u001b[39;00m\n\u001b[1;32m 262\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 263\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01murllib\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mrequest\u001b[39;00m\n\u001b[0;32m--> 265\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43murllib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:216\u001b[0m, in \u001b[0;36murlopen\u001b[0;34m(url, data, timeout, cafile, capath, cadefault, context)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 215\u001b[0m opener \u001b[38;5;241m=\u001b[39m _opener\n\u001b[0;32m--> 216\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mopener\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:519\u001b[0m, in \u001b[0;36mOpenerDirector.open\u001b[0;34m(self, fullurl, data, timeout)\u001b[0m\n\u001b[1;32m 516\u001b[0m req \u001b[38;5;241m=\u001b[39m meth(req)\n\u001b[1;32m 518\u001b[0m sys\u001b[38;5;241m.\u001b[39maudit(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124murllib.Request\u001b[39m\u001b[38;5;124m'\u001b[39m, req\u001b[38;5;241m.\u001b[39mfull_url, req\u001b[38;5;241m.\u001b[39mdata, req\u001b[38;5;241m.\u001b[39mheaders, req\u001b[38;5;241m.\u001b[39mget_method())\n\u001b[0;32m--> 519\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 521\u001b[0m \u001b[38;5;66;03m# post-process response\u001b[39;00m\n\u001b[1;32m 522\u001b[0m meth_name \u001b[38;5;241m=\u001b[39m protocol\u001b[38;5;241m+\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_response\u001b[39m\u001b[38;5;124m\"\u001b[39m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:536\u001b[0m, in \u001b[0;36mOpenerDirector._open\u001b[0;34m(self, req, data)\u001b[0m\n\u001b[1;32m 533\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n\u001b[1;32m 535\u001b[0m protocol \u001b[38;5;241m=\u001b[39m req\u001b[38;5;241m.\u001b[39mtype\n\u001b[0;32m--> 536\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_chain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhandle_open\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprotocol\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\n\u001b[1;32m 537\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m_open\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 538\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result:\n\u001b[1;32m 539\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:496\u001b[0m, in \u001b[0;36mOpenerDirector._call_chain\u001b[0;34m(self, chain, kind, meth_name, *args)\u001b[0m\n\u001b[1;32m 494\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handler \u001b[38;5;129;01min\u001b[39;00m handlers:\n\u001b[1;32m 495\u001b[0m func \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mgetattr\u001b[39m(handler, meth_name)\n\u001b[0;32m--> 496\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 497\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m result \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 498\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:1391\u001b[0m, in \u001b[0;36mHTTPSHandler.https_open\u001b[0;34m(self, req)\u001b[0m\n\u001b[1;32m 1390\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mhttps_open\u001b[39m(\u001b[38;5;28mself\u001b[39m, req):\n\u001b[0;32m-> 1391\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_open\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhttp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mHTTPSConnection\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1392\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcheck_hostname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_check_hostname\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/anaconda3/lib/python3.10/urllib/request.py:1351\u001b[0m, in \u001b[0;36mAbstractHTTPHandler.do_open\u001b[0;34m(self, http_class, req, **http_conn_args)\u001b[0m\n\u001b[1;32m 1348\u001b[0m h\u001b[38;5;241m.\u001b[39mrequest(req\u001b[38;5;241m.\u001b[39mget_method(), req\u001b[38;5;241m.\u001b[39mselector, req\u001b[38;5;241m.\u001b[39mdata, headers,\n\u001b[1;32m 1349\u001b[0m encode_chunked\u001b[38;5;241m=\u001b[39mreq\u001b[38;5;241m.\u001b[39mhas_header(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTransfer-encoding\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[1;32m 1350\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err: \u001b[38;5;66;03m# timeout error\u001b[39;00m\n\u001b[0;32m-> 1351\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m URLError(err)\n\u001b[1;32m 1352\u001b[0m r \u001b[38;5;241m=\u001b[39m h\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[1;32m 1353\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n", - "\u001b[0;31mURLError\u001b[0m: " - ] - } - ], - "source": [ - "# There should be no errors here\n", - "dataset = os.path.join(DATA_PATH, \"SIR_data_case_hosp.csv\")\n", - "load_data(dataset, data_mapping) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "840ce831", - "metadata": {}, - "outputs": [], - "source": [ - "# Should throw an error for missing data\n", - "dataset = os.path.join(DATA_PATH, \"SIR_missing_data.csv\")\n", - "load_data(dataset, data_mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d8d07064", - "metadata": {}, - "outputs": [], - "source": [ - "# Should throw an error for NaN-type data\n", - "dataset = os.path.join(DATA_PATH, \"SIR_data_nan.csv\")\n", - "load_data(dataset, data_mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26c24d2f", - "metadata": {}, - "outputs": [], - "source": [ - "# Should throw an error for missing data in the form of a space\n", - "dataset = os.path.join(DATA_PATH, \"SIR_missing_data_space.csv\")\n", - "load_data(dataset, data_mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "583d7b4a", - "metadata": {}, - "outputs": [], - "source": [ - "# Should throw an error for None-type data\n", - "dataset = os.path.join(DATA_PATH, \"SIR_data_none_type.csv\")\n", - "load_data(dataset, data_mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4d417aab", - "metadata": {}, - "outputs": [], - "source": [ - "# Should throw an error for having the wrong column name\n", - "dataset = os.path.join(DATA_PATH, \"SIR_data_wrong_time_col_name.csv\")\n", - "load_data(dataset, data_mapping)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "64badecc", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "298ee978", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python (pyciemss)", - "language": "python", - "name": "pyciemss" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/pyciemss/integration_utils/observation.py b/pyciemss/integration_utils/observation.py index 6736119e0..cbd19edd0 100644 --- a/pyciemss/integration_utils/observation.py +++ b/pyciemss/integration_utils/observation.py @@ -1,4 +1,4 @@ -from typing import Dict, Tuple +from typing import Dict, Tuple, Union import pandas as pd import torch @@ -9,7 +9,7 @@ def load_data( - path: str, data_mapping: Dict[str, str] = {} + path: Union[str, pd.DataFrame], data_mapping: Dict[str, str] = {} ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: """ Load data from a CSV file. @@ -21,29 +21,52 @@ def load_data( - If not provided, we will assume that the column names in the data file match the state variable names. """ - def check_data(data_path: str): - # This function checks a dataset formatting errors, and returns a dataframe - data_df = pd.read_csv(path) - print(data_df.head()) + def check_data(data_path: Union[str, pd.DataFrame]): + # This function checks a dataset for formatting errors, and returns a DataFrame + + # Read the data + if isinstance(data_path, str): + # If data_path is a string, assume it's a file path and read as a csv + data_df = pd.read_csv(data_path) + elif isinstance(data_path, pd.DataFrame): + # If data_path is a DataFrame, use it directly + data_df = data_path + else: + # If data_path is neither a string nor a DataFrame, raise an error + raise ValueError("data_path must be either a file path or a DataFrame.") # Check that the first column name is "Timestamp" - assert ( - data_df.columns[0] == "Timestamp" - ), "The column of timepoints must be first, and named 'Timestamp'." + if data_df.columns[0] != "Timestamp": + raise ValueError( + "The first column must be named 'Timestamp' and contain the time corresponding to each row of data." + ) # Check that there are no NaN values or empty entries - assert ( - not data_df.isna().any().any() - ), "Dataset cannot contain NaN or empty entries" + if data_df.isna().any().any(): + raise ValueError("Dataset cannot contain NaN or empty entries.") # Check that there is no missing data in the form of None type or char values - assert ( - data_df.applymap(lambda x: isinstance(x, (int, float))).all().all() - ), "Dataset cannot contain None type or char values. All entries must be of type `int` or `float`." + if not data_df.applymap(lambda x: isinstance(x, (int, float))).all().all(): + raise ValueError( + "Dataset cannot contain None type or char values. All entries must be of type `int` or `float`." + ) return data_df + def print_dataframe_report(data_df): + # Prints a short report about the data + + print( + f"Data printout: This dataset contains {len(data_df) - 1} rows of data. The first column, {data_df.columns[0]}, begins at {data_df.iloc[0, 0]} and ends at {data_df.iloc[-1, 0]}. The subsequent columns are named: " + ) + + # Print the names of subsequent columns + other_col_names = data_df.columns[1:] + for column_name in other_col_names: + print(column_name) + df = check_data(path) + print_dataframe_report(df) data_timepoints = torch.tensor(df["Timestamp"].values, dtype=torch.float32) data = {} diff --git a/tests/fixtures.py b/tests/fixtures.py index 520d4b9c4..9a05d18cb 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -3,6 +3,7 @@ from typing import Any, Dict, Optional, TypeVar import numpy as np +import pandas as pd import torch from pyciemss.ouu.qoi import obs_nday_average_qoi @@ -107,6 +108,40 @@ def __init__( torch.tensor(3), ] # bad candidates for num_samples/num_iterations +bad_data1 = { + "Timestamp": {0: 1.1, 1: 2.2, 2: 3.3}, + "case": {0: 15.0, 1: "", 2: 20.0}, + "hosp": {0: 0.1, 1: 1.0, 2: 2.2}, +} +bad_data2 = { + "Timestamp": {0: 1.1, 1: 2.2, 2: 3.3}, + "case": {0: 15.0, 1: "apple", 2: 20.0}, + "hosp": {0: 0.1, 1: 1.0, 2: 2.2}, +} +bad_data3 = { + "Timestamp": {0: 1.1, 1: 2.2, 2: 3.3}, + "case": {0: 15.0, 1: " ", 2: 20.0}, + "hosp": {0: 0.1, 1: 1.0, 2: 2.2}, +} +bad_data4 = { + "Timestamp": {0: 1.1, 1: 2.2, 2: 3.3}, + "case": {0: 15.0, 1: None, 2: 20.0}, + "hosp": {0: 0.1, 1: 1.0, 2: 2.2}, +} +bad_data5 = { + "Timepoints": {0: 1.1, 1: 2.2, 2: 3.3}, + "case": {0: 15.0, 1: 18.0, 2: 20.0}, + "hosp": {0: 0.1, 1: 1.0, 2: 2.2}, +} +BADLY_FORMATTED_DATAFRAMES = [ + pd.DataFrame(bad_data1), + pd.DataFrame(bad_data2), + pd.DataFrame(bad_data3), + pd.DataFrame(bad_data4), + pd.DataFrame(bad_data5), +] # improperly formatted datasets +MAPPING_FOR_DATA_TESTS = {"case": "I", "hosp": "H"} + def check_keys_match(obj1: Dict[str, T], obj2: Dict[str, T]): assert set(obj1.keys()) == set(obj2.keys()), "Objects have different variables." diff --git a/tests/test_interfaces.py b/tests/test_interfaces.py index 0ef41cf1b..81bb948f9 100644 --- a/tests/test_interfaces.py +++ b/tests/test_interfaces.py @@ -9,8 +9,10 @@ from pyciemss.interfaces import calibrate, ensemble_sample, optimize, sample from .fixtures import ( + BADLY_FORMATTED_DATAFRAMES, END_TIMES, LOGGING_STEP_SIZES, + MAPPING_FOR_DATA_TESTS, MODEL_URLS, MODELS, NON_POS_INTS, @@ -516,3 +518,14 @@ def test_non_pos_int_sample( logging_step_size, num_samples=bad_num_samples, ) + + +@pytest.mark.parametrize("bad_data", BADLY_FORMATTED_DATAFRAMES) +@pytest.mark.parametrize("data_mapping", MAPPING_FOR_DATA_TESTS) +def test_load_data(bad_data, data_mapping): + # Assert that a ValueError is raised for improperly formatted data + with pytest.raises(ValueError): + load_data( + bad_data, + data_mapping, + ) From 506922033ef636d93d4b8826310720b59154bff8 Mon Sep 17 00:00:00 2001 From: sabinala <130604122+sabinala@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:09:06 -0800 Subject: [PATCH 5/7] Update observation.py --- pyciemss/integration_utils/observation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyciemss/integration_utils/observation.py b/pyciemss/integration_utils/observation.py index cbd19edd0..227464dff 100644 --- a/pyciemss/integration_utils/observation.py +++ b/pyciemss/integration_utils/observation.py @@ -53,7 +53,7 @@ def check_data(data_path: Union[str, pd.DataFrame]): return data_df - def print_dataframe_report(data_df): + def print_data_report(data_df): # Prints a short report about the data print( @@ -66,7 +66,7 @@ def print_dataframe_report(data_df): print(column_name) df = check_data(path) - print_dataframe_report(df) + print_data_report(df) data_timepoints = torch.tensor(df["Timestamp"].values, dtype=torch.float32) data = {} From 5410c2a375b61e0b9380f45526ebf3a1f6fb7a5d Mon Sep 17 00:00:00 2001 From: sabinala <130604122+sabinala@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:10:24 -0800 Subject: [PATCH 6/7] Update observation.py --- pyciemss/integration_utils/observation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyciemss/integration_utils/observation.py b/pyciemss/integration_utils/observation.py index 227464dff..86979bafa 100644 --- a/pyciemss/integration_utils/observation.py +++ b/pyciemss/integration_utils/observation.py @@ -12,9 +12,9 @@ def load_data( path: Union[str, pd.DataFrame], data_mapping: Dict[str, str] = {} ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: """ - Load data from a CSV file. + Load data from a CSV file, or directly from a DataFrame. - - path: path to the CSV file + - path: path to the CSV file, or DataFrame - data_mapping: A mapping from column names in the data file to state variable names in the model. - keys: str name of column in dataset - values: str name of state/observable in model From a59d102e64dc89f9ea2be8d1b4ddea311f596fce Mon Sep 17 00:00:00 2001 From: sabinala <130604122+sabinala@users.noreply.github.com> Date: Wed, 21 Feb 2024 12:48:45 -0800 Subject: [PATCH 7/7] Update observation.py Cleaning up data report --- pyciemss/integration_utils/observation.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pyciemss/integration_utils/observation.py b/pyciemss/integration_utils/observation.py index 86979bafa..f48061f28 100644 --- a/pyciemss/integration_utils/observation.py +++ b/pyciemss/integration_utils/observation.py @@ -57,14 +57,13 @@ def print_data_report(data_df): # Prints a short report about the data print( - f"Data printout: This dataset contains {len(data_df) - 1} rows of data. The first column, {data_df.columns[0]}, begins at {data_df.iloc[0, 0]} and ends at {data_df.iloc[-1, 0]}. The subsequent columns are named: " + f"Data printout: This dataset contains {len(data_df) - 1} rows of data. " + f"The first column, {data_df.columns[0]}, begins at {data_df.iloc[0, 0]} " + f"and ends at {data_df.iloc[-1, 0]}. " + f"The subsequent columns are named: " + f"{', '.join(data_df.columns[1:])}" ) - # Print the names of subsequent columns - other_col_names = data_df.columns[1:] - for column_name in other_col_names: - print(column_name) - df = check_data(path) print_data_report(df)