feat(electric): Add support for DATE data type in electrified tables (#…

…349)

Closes VAX-855.

.changeset/forty-actors-sin.md

@@ -0,0 +1,5 @@
+---
+"@core/electric": patch
+---
+
+Implement support for the DATE column type in electrified tables.

clients/typescript/src/satellite/client.ts

@@ -1046,6 +1046,7 @@ function deserializeColumnData(
   const columnType = columnInfo.type.toUpperCase()
   switch (columnType) {
     case 'CHAR':
+    case 'DATE':
     case 'TEXT':
     case 'TIMESTAMP':
     case 'TIMESTAMPTZ':

components/electric/lib/electric/satellite/serialization.ex

@@ -24,8 +24,9 @@ defmodule Electric.Satellite.Serialization do
   @spec supported_pg_types :: [atom]
   def supported_pg_types do
     ~w[
-      int2 int4 int8
+      date
       float8
+      int2 int4 int8
       text
       timestamp timestamptz
       uuid
@@ -480,11 +481,19 @@ defmodule Electric.Satellite.Serialization do
     val
   end
 
-  def decode_column_value!(val, type) when type in [:int2, :int4, :int8] do
-    :ok =
-      val
-      |> String.to_integer()
-      |> assert_integer_in_range!(type)
+  def decode_column_value!(val, :date) do
+    <<yyyy::binary-4, ?-, mm::binary-2, ?-, dd::binary-2>> = val
+
+    year = String.to_integer(yyyy)
+    :ok = assert_year_in_range(year)
+
+    month = String.to_integer(mm)
+    true = month in 1..12
+
+    day = String.to_integer(dd)
+    true = day in 1..31
+
+    %Date{} = Date.from_iso8601!(val)
 
     val
   end
@@ -494,6 +503,15 @@ defmodule Electric.Satellite.Serialization do
     val
   end
 
+  def decode_column_value!(val, type) when type in [:int2, :int4, :int8] do
+    :ok =
+      val
+      |> String.to_integer()
+      |> assert_integer_in_range!(type)
+
+    val
+  end
+
   def decode_column_value!(val, :timestamp) do
     # NaiveDateTime silently discards time zone offset if it is present in the string. But we want to reject such strings
     # because values of type `timestamp` must not have an offset.

components/electric/test/electric/postgres/extension_test.exs

@@ -439,7 +439,8 @@ defmodule Electric.Postgres.ExtensionTest do
                  real8a FLOAT8,
                  real8b DOUBLE PRECISION,
                  ts TIMESTAMP,
-                 tstz TIMESTAMPTZ
+                 tstz TIMESTAMPTZ,
+                 d DATE
                );
                CALL electric.electrify('public.t1');
                """)

components/electric/test/electric/satellite/serialization_test.exs

@@ -18,7 +18,8 @@ defmodule Electric.Satellite.SerializationTest do
         "int" => "13",
         "var" => "...",
         "real" => "-3.14",
-        "id" => uuid
+        "id" => uuid,
+        "date" => "2024-12-24"
       }
 
       columns = [
@@ -28,11 +29,12 @@ defmodule Electric.Satellite.SerializationTest do
         %{name: "id", type: :uuid},
         %{name: "int", type: :int4},
         %{name: "var", type: :varchar},
-        %{name: "real", type: :float8}
+        %{name: "real", type: :float8},
+        %{name: "date", type: :date}
       ]
 
       assert %SatOpRow{
-               values: ["", "", "4", uuid, "13", "...", "-3.14"],
+               values: ["", "", "4", uuid, "13", "...", "-3.14", "2024-12-24"],
                nulls_bitmask: <<0b11000000>>
              } == Serialization.map_to_row(data, columns)
     end
@@ -64,7 +66,7 @@ defmodule Electric.Satellite.SerializationTest do
   describe "decode_record!" do
     test "decodes a SatOpRow struct into a map" do
       row = %SatOpRow{
-        nulls_bitmask: <<0b00100001>>,
+        nulls_bitmask: <<0b00100001, 0>>,
         values: [
           "256",
           "hello",
@@ -73,7 +75,8 @@ defmodule Electric.Satellite.SerializationTest do
           "-1.0e124",
           "2023-08-15 17:20:31",
           "2023-08-15 17:20:31Z",
-          ""
+          "",
+          "0400-02-29"
         ]
       }
 
@@ -85,7 +88,8 @@ defmodule Electric.Satellite.SerializationTest do
         %{name: "real2", type: :float8},
         %{name: "t", type: :timestamp},
         %{name: "tz", type: :timestamptz},
-        %{name: "x", type: :float4, nullable?: true}
+        %{name: "x", type: :float4, nullable?: true},
+        %{name: "date", type: :date}
       ]
 
       assert %{
@@ -96,7 +100,8 @@ defmodule Electric.Satellite.SerializationTest do
                "real2" => "-1.0e124",
                "t" => "2023-08-15 17:20:31",
                "tz" => "2023-08-15 17:20:31Z",
-               "x" => nil
+               "x" => nil,
+               "date" => "0400-02-29"
              } == Serialization.decode_record!(row, columns)
     end
 
@@ -117,8 +122,17 @@ defmodule Electric.Satellite.SerializationTest do
         {"2023-08-15 11:12:13+01", :timestamptz},
         {"2023-08-15 11:12:13+99:98", :timestamptz},
         {"2023-08-15 11:12:13+00", :timestamptz},
+        {"2023-08-15 11:12:13", :timestamptz},
         {"0000-08-15 23:00:00Z", :timestamptz},
-        {"-2000-08-15 23:00:00Z", :timestamptz}
+        {"-2000-08-15 23:00:00Z", :timestamptz},
+        {"0000-01-01", :date},
+        {"005-01-01", :date},
+        {"05-01-01", :date},
+        {"9-01-01", :date},
+        {"1999-31-12", :date},
+        {"20230815", :date},
+        {"-2023-08-15", :date},
+        {"12-12-12", :date}
       ]
 
       Enum.each(test_data, fn {val, type} ->

components/electric/test/electric/satellite/ws_validations_test.exs

@@ -223,6 +223,49 @@ defmodule Electric.Satellite.WsValidationsTest do
     end)
   end
 
+  test "validates date values", ctx do
+    vsn = "2023082201"
+
+    :ok =
+      migrate(ctx.db, vsn, "public.foo", "CREATE TABLE public.foo (id TEXT PRIMARY KEY, d date)")
+
+    valid_records = [
+      %{"id" => "1", "d" => "2023-08-07"},
+      %{"id" => "2", "d" => "5697-02-28"},
+      %{"id" => "3", "d" => "6000-02-29"},
+      %{"id" => "4", "d" => "0001-01-01"}
+    ]
+
+    within_replication_context(ctx, vsn, fn conn ->
+      Enum.each(valid_records, fn record ->
+        tx_op_log = serialize_trans(record)
+        MockClient.send_data(conn, tx_op_log)
+      end)
+    end)
+
+    refute_receive {_, %SatErrorResp{error_type: :INVALID_REQUEST}}, @receive_timeout
+
+    invalid_records = [
+      %{"id" => "10", "d" => "now"},
+      %{"id" => "11", "d" => "today"},
+      %{"id" => "12", "d" => "20230822"},
+      %{"id" => "13", "d" => "22-08-2023"},
+      %{"id" => "14", "d" => "2023-22-08"},
+      %{"id" => "15", "d" => "-1999-01-01"},
+      %{"id" => "16", "d" => "001-01-01"},
+      %{"id" => "17", "d" => "2000-13-13"},
+      %{"id" => "18", "d" => "5697-02-29"}
+    ]
+
+    Enum.each(invalid_records, fn record ->
+      within_replication_context(ctx, vsn, fn conn ->
+        tx_op_log = serialize_trans(record)
+        MockClient.send_data(conn, tx_op_log)
+        assert_receive {^conn, %SatErrorResp{error_type: :INVALID_REQUEST}}, @receive_timeout
+      end)
+    end)
+  end
+
   test "validates timestamp values", ctx do
     vsn = "2023072505"
 

e2e/tests/03.14_node_satellite_can_sync_dates.lux

@@ -0,0 +1,72 @@
+[doc NodeJS Satellite correctly syncs DATE values from and to Electric]
+[include _shared.luxinc]
+[include _satellite_macros.luxinc]
+
+[invoke setup]
+
+[shell pg_1]
+    [local sql=
+        """
+        CREATE TABLE public.dates (
+            id TEXT PRIMARY KEY DEFAULT uuid_generate_v4(),
+            d DATE
+        );
+        CALL electric.electrify('public.dates');
+        """]
+    [invoke migrate_pg 20230823 $sql]
+
+[invoke setup_client 1 electric_1 5133]
+
+[shell satellite_1]
+    [invoke node_await_table "dates"]
+    [invoke node_sync_table "dates"]
+
+[shell pg_1]
+    !INSERT INTO public.dates (id, d) VALUES ('001', '2023-08-23'), ('002', '01-01-0001'), ('003', 'Feb 29 6000');
+    ??INSERT 0 3
+
+[shell satellite_1]
+    [invoke node_await_get_from_table "dates" "003"]
+
+    ??id: '001'
+    ??d: '2023-08-23'
+
+    ??id: '002'
+    ??d: '0001-01-01'
+
+    ??id: '003'
+    ??d: '6000-02-29'
+
+    [invoke node_await_insert_extended_into "dates" "{id: '004', d: '1999-12-31'}"]
+
+[shell pg_1]
+    [invoke wait-for "SELECT * FROM public.dates;" "004" 10 $psql]
+
+    !SELECT id, d FROM public.dates;
+    ??001 | 2023-08-23
+    ??002 | 0001-01-01
+    ??003 | 6000-02-29
+    ??004 | 1999-12-31
+
+# Start a new Satellite client and verify that it receives all dates
+[invoke setup_client 2 electric_1 5133]
+
+[shell satellite_2]
+    [invoke node_await_table "dates"]
+    [invoke node_sync_table "dates"]
+
+    [invoke node_await_get_from_table "dates" "004"]
+    ??id: '001'
+    ??d: '2023-08-23'
+
+    ??id: '002'
+    ??d: '0001-01-01'
+
+    ??id: '003'
+    ??d: '6000-02-29'
+
+    ??id: '004'
+    ??d: '1999-12-31'
+
+[cleanup]
+  [invoke teardown]