From bd3ccf96155af915026559edf439c8c4257dda59 Mon Sep 17 00:00:00 2001
From: Micah Sandusky <micc5725@gmail.com>
Date: Mon, 5 Aug 2024 15:36:52 -0600
Subject: [PATCH] More tests passing

---
 snowex_db/metadata.py | 19 +++++++++++++++----
 snowex_db/upload.py   |  3 ++-
 tests/test_batch.py   |  5 ++++-
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/snowex_db/metadata.py b/snowex_db/metadata.py
index 650a17e..2017e81 100644
--- a/snowex_db/metadata.py
+++ b/snowex_db/metadata.py
@@ -158,9 +158,14 @@ def _read(self, filename):
         str_cols = remap_data_names(str_cols, DataHeader.rename)
 
         dtype = {k: str for k in str_cols}
-        df = pd.read_csv(filename, header=header_pos, names=str_cols,
-                         usecols=range(n_cols), encoding='latin',
-                         parse_dates=[0], dtype=dtype)
+        df = pd.read_csv(
+            filename, header=header_pos, names=str_cols,
+            usecols=range(n_cols), encoding='latin',
+            # parse_dates=[0],
+            dtype=dtype
+        )
+        # WHY IS THIS NEEDED?
+        df["date"] = pd.to_datetime(df["date"])
 
         # Insure all values are 4 digits. Seems like some were not by accident
         df['fname_sufix'] = df['fname_sufix'].apply(lambda v: v.zfill(4))
@@ -286,7 +291,7 @@ class ExtendedSnowExProfileVariables(SnowExProfileVariables):
     )
     IGNORE = MeasurementDescription(
         "ignore", "Ignore this",
-        ["original_index", 'id', 'freq_mhz', 'camera']
+        ["original_index", 'id', 'freq_mhz', 'camera', 'avgvelocity']
     )
     SAMPLE_SIGNAL = MeasurementDescription(
         'sample_signal', "Sample Signal",
@@ -302,6 +307,12 @@ class ExtendedSnowExProfileVariables(SnowExProfileVariables):
         'specific_surface_area', "Specific Surface Area",
         ['specific_surface_area']
     )
+    DATETIME = MeasurementDescription(
+        'datetime', "Combined date and time",
+        ["Date/Local Standard Time", "date/local_standard_time", "datetime",
+         "date&time"],
+        True
+    )
     DATE = MeasurementDescription(
         'date', "Measurement Date (only date column)",
         ['date_dd_mmm_yy', 'date']
diff --git a/snowex_db/upload.py b/snowex_db/upload.py
index 43097c0..4c7cf12 100644
--- a/snowex_db/upload.py
+++ b/snowex_db/upload.py
@@ -173,7 +173,8 @@ def build_data(self, data_name):
 
         # Assign all meta data to every entry to the data frame
         for k, v in self.hdr.info.items():
-            df[k] = v
+            if not pd.isna(v):
+                df[k] = v
 
         df['type'] = data_name
         df['date_accessed'] = self.date_accessed
diff --git a/tests/test_batch.py b/tests/test_batch.py
index 67352f9..38afaea 100644
--- a/tests/test_batch.py
+++ b/tests/test_batch.py
@@ -121,7 +121,10 @@ class TestUploadSMPBatch(TableTestBase):
     Test whether we can assign meta info from an smp log to 2 profiles
     """
     args = [['S19M1013_5S21_20200201.CSV', 'S06M0874_2N12_20200131.CSV']]
-    kwargs = {'in_timezone': 'UTC', 'smp_log_f': 'smp_log.csv', 'units': 'Newtons'}
+    kwargs = {
+        'in_timezone': 'UTC',
+        'smp_log_f': 'smp_log.csv',
+        'units': 'Newtons'}
     UploaderClass = UploadProfileBatch
     TableClass = LayerData
     attribute = 'depth'