fix: make sure all timestamps are timestamps with tzs

rust/cli/tests/cli_test.rs

@@ -332,10 +332,9 @@ async fn test_duckdb_snapshots() {
         // Check that quary_valid_from has the same date as the current date
         let current_date = Utc::now().date_naive();
         let quary_valid_from_str = &result.rows[0][4];
-        let quary_valid_from_date = quary_valid_from_str.split_whitespace().next().unwrap();
-        println!("quary_valid_from_date: {}", quary_valid_from_date);
         let quary_valid_from =
-            chrono::NaiveDate::parse_from_str(quary_valid_from_date, "%Y-%m-%dT%H:%M:%SZ").unwrap();
+            chrono::NaiveDate::parse_from_str(quary_valid_from_str, "%Y-%m-%d %H:%M:%S%.6f %Z")
+                .unwrap();
         assert_eq!(current_date, quary_valid_from);
 
         // Update orders.csv data
@@ -378,16 +377,13 @@ async fn test_duckdb_snapshots() {
         assert_eq!(updated_result.rows[1][5], "NULL"); // quary_valid_to should be NULL
 
         // Check that quary_valid_from of the updated row has the same date as the current date
+        // Check that quary_valid_from has the same date as the current date
         let current_date = Utc::now().date_naive();
-        let updated_quary_valid_from_str = &updated_result.rows[1][4];
-        let updated_quary_valid_from_date = updated_quary_valid_from_str
-            .split_whitespace()
-            .next()
-            .unwrap();
-        let updated_quary_valid_from =
-            chrono::NaiveDate::parse_from_str(updated_quary_valid_from_date, "%Y-%m-%dT%H:%M:%SZ")
+        let quary_valid_from_str = &updated_result.rows[1][4];
+        let quary_valid_from =
+            chrono::NaiveDate::parse_from_str(quary_valid_from_str, "%Y-%m-%d %H:%M:%S%.6f %Z")
                 .unwrap();
-        assert_eq!(current_date, updated_quary_valid_from);
+        assert_eq!(current_date, quary_valid_from);
     }
 }
 

rust/core/src/database_duckdb.rs

@@ -30,7 +30,10 @@ impl DatabaseQueryGeneratorDuckDB {
             .override_now
             .map(|time| -> DateTime<Utc> { time.into() })
             .unwrap_or(SystemTime::now().into());
-        format!("'{}'", datetime.format("%Y-%m-%dT%H:%M:%SZ"))
+        format!(
+            "CAST ('{}' AS TIMESTAMP WITH TIME ZONE)",
+            datetime.format("%Y-%m-%dT%H:%M:%SZ")
+        )
     }
 }
 
@@ -128,15 +131,15 @@ impl SnapshotGenerator for DatabaseQueryGeneratorDuckDB {
                     ) AS source
                     WHERE target.{unique_key} = source.{unique_key}
                         AND target.quary_valid_to IS NULL
-                        AND CAST(source.{updated_at} AS TIMESTAMP) > CAST(target.{updated_at} AS TIMESTAMP)"
+                        AND CAST(source.{updated_at} AS TIMESTAMP WITH TIME ZONE) > CAST(target.{updated_at} AS TIMESTAMP WITH TIME ZONE)"
                 );
 
                 let insert_sql = format!(
                     "INSERT INTO {path}
                     SELECT
                         *,
                         {now} AS quary_valid_from,
-                        NULL AS quary_valid_to,
+                        CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS quary_valid_to,
                         MD5(CAST(CONCAT({unique_key}, CAST({updated_at} AS STRING)) AS STRING)) AS quary_scd_id
                     FROM ({templated_select}) AS source
                     WHERE NOT EXISTS (
@@ -176,7 +179,6 @@ impl SnapshotGenerator for DatabaseQueryGeneratorDuckDB {
 
 #[cfg(test)]
 mod tests {
-
     use super::*;
 
     #[test]
@@ -213,10 +215,14 @@ mod tests {
         let override_now = SystemTime::now();
         let database = DatabaseQueryGeneratorDuckDB::new(None, Some(override_now));
 
+        // TODO Improve test
         let result = database.get_now();
         let expected_datetime: DateTime<Utc> = override_now.into();
-        let expected_result = format!("'{}'", expected_datetime.format("%Y-%m-%dT%H:%M:%SZ"));
-        assert_eq!(result, expected_result);
+        let expected_result = format!(
+            "CAST ('{}' AS TIMESTAMP WITH TIME ZONE)",
+            expected_datetime.format("%Y-%m-%dT%H:%M:%SZ")
+        );
+        assert_eq!(expected_result, result);
     }
 
     #[test]
@@ -242,6 +248,9 @@ mod tests {
             .generate_snapshot_sql(path, templated_select, unique_key, &strategy, None)
             .unwrap();
 
-        assert_eq!(result.iter().map(|s| s.as_str()).collect::<Vec<&str>>(), vec!["CREATE TABLE IF NOT EXISTS mytable AS (\n                      SELECT\n                        *,\n                        '2021-01-01T00:00:00Z' AS quary_valid_from,\n                        CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS quary_valid_to,\n                        MD5(CAST(CONCAT(id, CAST(updated_at AS STRING)) AS STRING)) AS quary_scd_id\n                    FROM (SELECT * FROM mytable)\n                    )", "UPDATE mytable AS target\n                    SET quary_valid_to = source.quary_valid_from\n                    FROM (\n                        SELECT\n                            *,\n                            '2021-01-01T00:00:00Z' AS quary_valid_from,\n                            MD5(CAST(CONCAT(id, CAST(updated_at AS STRING)) AS STRING)) AS quary_scd_id\n                        FROM (SELECT * FROM mytable)\n                    ) AS source\n                    WHERE target.id = source.id\n                        AND target.quary_valid_to IS NULL\n                        AND CAST(source.updated_at AS TIMESTAMP) > CAST(target.updated_at AS TIMESTAMP)", "INSERT INTO mytable\n                    SELECT\n                        *,\n                        '2021-01-01T00:00:00Z' AS quary_valid_from,\n                        NULL AS quary_valid_to,\n                        MD5(CAST(CONCAT(id, CAST(updated_at AS STRING)) AS STRING)) AS quary_scd_id\n                    FROM (SELECT * FROM mytable) AS source\n                    WHERE NOT EXISTS (\n                        SELECT 1\n                        FROM mytable AS target\n                        WHERE target.quary_scd_id = MD5(CAST(CONCAT(source.id, CAST(source.updated_at AS STRING)) AS STRING))\n                    )"]);
+        assert_eq!(
+            vec!["CREATE TABLE IF NOT EXISTS mytable AS (\n                      SELECT\n                        *,\n                        CAST ('2021-01-01T00:00:00Z' AS TIMESTAMP WITH TIME ZONE) AS quary_valid_from,\n                        CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS quary_valid_to,\n                        MD5(CAST(CONCAT(id, CAST(updated_at AS STRING)) AS STRING)) AS quary_scd_id\n                    FROM (SELECT * FROM mytable)\n                    )", "UPDATE mytable AS target\n                    SET quary_valid_to = source.quary_valid_from\n                    FROM (\n                        SELECT\n                            *,\n                            CAST ('2021-01-01T00:00:00Z' AS TIMESTAMP WITH TIME ZONE) AS quary_valid_from,\n                            MD5(CAST(CONCAT(id, CAST(updated_at AS STRING)) AS STRING)) AS quary_scd_id\n                        FROM (SELECT * FROM mytable)\n                    ) AS source\n                    WHERE target.id = source.id\n                        AND target.quary_valid_to IS NULL\n                        AND CAST(source.updated_at AS TIMESTAMP WITH TIME ZONE) > CAST(target.updated_at AS TIMESTAMP WITH TIME ZONE)", "INSERT INTO mytable\n                    SELECT\n                        *,\n                        CAST ('2021-01-01T00:00:00Z' AS TIMESTAMP WITH TIME ZONE) AS quary_valid_from,\n                        CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS quary_valid_to,\n                        MD5(CAST(CONCAT(id, CAST(updated_at AS STRING)) AS STRING)) AS quary_scd_id\n                    FROM (SELECT * FROM mytable) AS source\n                    WHERE NOT EXISTS (\n                        SELECT 1\n                        FROM mytable AS target\n                        WHERE target.quary_scd_id = MD5(CAST(CONCAT(source.id, CAST(source.updated_at AS STRING)) AS STRING))\n                    )"],
+            result.iter().map(|s| s.as_str()).collect::<Vec<&str>>(),
+        );
     }
 }

rust/core/src/database_postgres.rs

@@ -206,7 +206,7 @@ impl SnapshotGenerator for DatabaseQueryGeneratorPostgres {
                     SELECT
                         *,
                         {now} AS quary_valid_from,
-                        NULL AS quary_valid_to,
+                        CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS quary_valid_to,
                         MD5(CAST(CONCAT({unique_key}, CAST({updated_at} AS TEXT)) AS TEXT)) AS quary_scd_id
                     FROM ({templated_select}) AS source
                     WHERE NOT EXISTS (
@@ -266,6 +266,4 @@ mod tests {
             "CAST('1970-01-01T00:00:00Z' AS TIMESTAMP WITH TIME ZONE)".to_string()
         );
     }
-
-    // TODO MAKE SURE WE HAVE A TEST for sql generation
 }

rust/core/src/database_redshift.rs

@@ -206,7 +206,7 @@ impl SnapshotGenerator for DatabaseQueryGeneratorRedshift {
                     SELECT
                         *,
                         {now} AS quary_valid_from,
-                        NULL AS quary_valid_to,
+                        CAST(NULL AS TIMESTAMP WITH TIME ZONE) AS quary_valid_to,
                         MD5(CAST(CONCAT({unique_key}, CAST({updated_at} AS TEXT)) AS TEXT)) AS quary_scd_id
                     FROM ({templated_select}) AS source
                     WHERE NOT EXISTS (

rust/core/src/databases.rs

@@ -290,7 +290,8 @@ impl SnapshotGenerator for Box<dyn DatabaseQueryGenerator> {
         strategy: &StrategyType,
         now: &str,
     ) -> Result<String, String> {
-        self.as_ref().generate_snapshot_query(templated_select, unique_key, strategy, now)
+        self.as_ref()
+            .generate_snapshot_query(templated_select, unique_key, strategy, now)
     }
 }