diff --git a/.changeset/shy-days-whisper.md b/.changeset/shy-days-whisper.md new file mode 100644 index 0000000000..8baf362df2 --- /dev/null +++ b/.changeset/shy-days-whisper.md @@ -0,0 +1,5 @@ +--- +"@core/electric": patch +--- + +Implement support for the TIME column type in electrified tables. diff --git a/clients/typescript/src/satellite/client.ts b/clients/typescript/src/satellite/client.ts index 3287cc5f6d..d9dcc80851 100644 --- a/clients/typescript/src/satellite/client.ts +++ b/clients/typescript/src/satellite/client.ts @@ -1048,6 +1048,7 @@ function deserializeColumnData( case 'CHAR': case 'DATE': case 'TEXT': + case 'TIME': case 'TIMESTAMP': case 'TIMESTAMPTZ': case 'UUID': diff --git a/components/electric/lib/electric/satellite/serialization.ex b/components/electric/lib/electric/satellite/serialization.ex index 70f4cf4037..ad1c0df6d9 100644 --- a/components/electric/lib/electric/satellite/serialization.ex +++ b/components/electric/lib/electric/satellite/serialization.ex @@ -28,6 +28,7 @@ defmodule Electric.Satellite.Serialization do float8 int2 int4 int8 text + time timestamp timestamptz uuid varchar @@ -512,6 +513,25 @@ defmodule Electric.Satellite.Serialization do val end + def decode_column_value!(val, :time) do + <> <> frac = val + + hours = String.to_integer(hh) + true = hours in 0..23 + + minutes = String.to_integer(mm) + true = minutes in 0..59 + + seconds = String.to_integer(ss) + true = seconds in 0..59 + + :ok = validate_fractional_seconds(frac) + + _ = Time.from_iso8601!(val) + + val + end + def decode_column_value!(val, :timestamp) do # NaiveDateTime silently discards time zone offset if it is present in the string. But we want to reject such strings # because values of type `timestamp` must not have an offset. @@ -559,4 +579,13 @@ defmodule Electric.Satellite.Serialization do # [1]: https://www.postgresql.org/docs/current/datatype-datetime.html # [2]: https://www.sqlite.org/lang_datefunc.html defp assert_year_in_range(year) when year in 1..9999, do: :ok + + defp validate_fractional_seconds(""), do: :ok + + defp validate_fractional_seconds("." <> fs_str) do + # Fractional seconds must not exceed 6 decimal digits, otherwise Postgres will round the last digit up or down. + true = byte_size(fs_str) <= 6 + _ = String.to_integer(fs_str) + :ok + end end diff --git a/components/electric/test/electric/postgres/extension_test.exs b/components/electric/test/electric/postgres/extension_test.exs index faa5ea9a85..9c5150e09f 100644 --- a/components/electric/test/electric/postgres/extension_test.exs +++ b/components/electric/test/electric/postgres/extension_test.exs @@ -440,7 +440,8 @@ defmodule Electric.Postgres.ExtensionTest do real8b DOUBLE PRECISION, ts TIMESTAMP, tstz TIMESTAMPTZ, - d DATE + d DATE, + t TIME ); CALL electric.electrify('public.t1'); """) diff --git a/components/electric/test/electric/satellite/serialization_test.exs b/components/electric/test/electric/satellite/serialization_test.exs index af1a5da86f..f4bbc82505 100644 --- a/components/electric/test/electric/satellite/serialization_test.exs +++ b/components/electric/test/electric/satellite/serialization_test.exs @@ -19,7 +19,8 @@ defmodule Electric.Satellite.SerializationTest do "var" => "...", "real" => "-3.14", "id" => uuid, - "date" => "2024-12-24" + "date" => "2024-12-24", + "time" => "12:01:00.123" } columns = [ @@ -30,12 +31,13 @@ defmodule Electric.Satellite.SerializationTest do %{name: "int", type: :int4}, %{name: "var", type: :varchar}, %{name: "real", type: :float8}, - %{name: "date", type: :date} + %{name: "date", type: :date}, + %{name: "time", type: :time} ] assert %SatOpRow{ - values: ["", "", "4", uuid, "13", "...", "-3.14", "2024-12-24"], - nulls_bitmask: <<0b11000000>> + values: ["", "", "4", uuid, "13", "...", "-3.14", "2024-12-24", "12:01:00.123"], + nulls_bitmask: <<0b11000000, 0>> } == Serialization.map_to_row(data, columns) end @@ -76,7 +78,8 @@ defmodule Electric.Satellite.SerializationTest do "2023-08-15 17:20:31", "2023-08-15 17:20:31Z", "", - "0400-02-29" + "0400-02-29", + "03:59:59" ] } @@ -89,7 +92,8 @@ defmodule Electric.Satellite.SerializationTest do %{name: "t", type: :timestamp}, %{name: "tz", type: :timestamptz}, %{name: "x", type: :float4, nullable?: true}, - %{name: "date", type: :date} + %{name: "date", type: :date}, + %{name: "time", type: :time} ] assert %{ @@ -101,7 +105,8 @@ defmodule Electric.Satellite.SerializationTest do "t" => "2023-08-15 17:20:31", "tz" => "2023-08-15 17:20:31Z", "x" => nil, - "date" => "0400-02-29" + "date" => "0400-02-29", + "time" => "03:59:59" } == Serialization.decode_record!(row, columns) end @@ -132,7 +137,17 @@ defmodule Electric.Satellite.SerializationTest do {"1999-31-12", :date}, {"20230815", :date}, {"-2023-08-15", :date}, - {"12-12-12", :date} + {"12-12-12", :date}, + {"24:00:00", :time}, + {"-12:00:00", :time}, + {"22:01", :time}, + {"02:60:00", :time}, + {"02:00:60", :time}, + {"1:2:3", :time}, + {"010203", :time}, + {"016003", :time}, + {"00:00:00.", :time}, + {"00:00:00.1234567", :time} ] Enum.each(test_data, fn {val, type} -> diff --git a/components/electric/test/electric/satellite/ws_validations_test.exs b/components/electric/test/electric/satellite/ws_validations_test.exs index 40d79e0eee..bc0d2f4796 100644 --- a/components/electric/test/electric/satellite/ws_validations_test.exs +++ b/components/electric/test/electric/satellite/ws_validations_test.exs @@ -266,6 +266,51 @@ defmodule Electric.Satellite.WsValidationsTest do end) end + test "validates time values", ctx do + vsn = "2023091101" + + :ok = + migrate(ctx.db, vsn, "public.foo", "CREATE TABLE public.foo (id TEXT PRIMARY KEY, t time)") + + valid_records = [ + %{"id" => "1", "t" => "00:00:00"}, + %{"id" => "2", "t" => "23:59:59"}, + %{"id" => "3", "t" => "00:00:00.332211"}, + %{"id" => "4", "t" => "11:11:11.11"} + ] + + within_replication_context(ctx, vsn, fn conn -> + Enum.each(valid_records, fn record -> + tx_op_log = serialize_trans(record) + MockClient.send_data(conn, tx_op_log) + end) + end) + + refute_receive {_, %SatErrorResp{error_type: :INVALID_REQUEST}}, @receive_timeout + + invalid_records = [ + %{"id" => "10", "t" => "now"}, + %{"id" => "11", "t" => "::"}, + %{"id" => "12", "t" => "20:12"}, + %{"id" => "13", "t" => "T18:00"}, + %{"id" => "14", "t" => "l2:o6:t0"}, + %{"id" => "15", "t" => "1:20:23"}, + %{"id" => "16", "t" => "02:02:03-08:00"}, + %{"id" => "17", "t" => "01:00:00+0"}, + %{"id" => "18", "t" => "99:99:99"}, + %{"id" => "19", "t" => "12:1:0"}, + %{"id" => "20", "t" => ""} + ] + + Enum.each(invalid_records, fn record -> + within_replication_context(ctx, vsn, fn conn -> + tx_op_log = serialize_trans(record) + MockClient.send_data(conn, tx_op_log) + assert_receive {^conn, %SatErrorResp{error_type: :INVALID_REQUEST}}, @receive_timeout + end) + end) + end + test "validates timestamp values", ctx do vsn = "2023072505" diff --git a/e2e/tests/03.14_node_satellite_can_sync_dates.lux b/e2e/tests/03.14_node_satellite_can_sync_dates.lux deleted file mode 100644 index 51eb73c4cb..0000000000 --- a/e2e/tests/03.14_node_satellite_can_sync_dates.lux +++ /dev/null @@ -1,72 +0,0 @@ -[doc NodeJS Satellite correctly syncs DATE values from and to Electric] -[include _shared.luxinc] -[include _satellite_macros.luxinc] - -[invoke setup] - -[shell pg_1] - [local sql= - """ - CREATE TABLE public.dates ( - id TEXT PRIMARY KEY DEFAULT uuid_generate_v4(), - d DATE - ); - CALL electric.electrify('public.dates'); - """] - [invoke migrate_pg 20230823 $sql] - -[invoke setup_client 1 electric_1 5133] - -[shell satellite_1] - [invoke node_await_table "dates"] - [invoke node_sync_table "dates"] - -[shell pg_1] - !INSERT INTO public.dates (id, d) VALUES ('001', '2023-08-23'), ('002', '01-01-0001'), ('003', 'Feb 29 6000'); - ??INSERT 0 3 - -[shell satellite_1] - [invoke node_await_get_from_table "dates" "003"] - - ??id: '001' - ??d: '2023-08-23' - - ??id: '002' - ??d: '0001-01-01' - - ??id: '003' - ??d: '6000-02-29' - - [invoke node_await_insert_extended_into "dates" "{id: '004', d: '1999-12-31'}"] - -[shell pg_1] - [invoke wait-for "SELECT * FROM public.dates;" "004" 10 $psql] - - !SELECT id, d FROM public.dates; - ??001 | 2023-08-23 - ??002 | 0001-01-01 - ??003 | 6000-02-29 - ??004 | 1999-12-31 - -# Start a new Satellite client and verify that it receives all dates -[invoke setup_client 2 electric_1 5133] - -[shell satellite_2] - [invoke node_await_table "dates"] - [invoke node_sync_table "dates"] - - [invoke node_await_get_from_table "dates" "004"] - ??id: '001' - ??d: '2023-08-23' - - ??id: '002' - ??d: '0001-01-01' - - ??id: '003' - ??d: '6000-02-29' - - ??id: '004' - ??d: '1999-12-31' - -[cleanup] - [invoke teardown] diff --git a/e2e/tests/03.14_node_satellite_can_sync_dates_and_times.lux b/e2e/tests/03.14_node_satellite_can_sync_dates_and_times.lux new file mode 100644 index 0000000000..7f7ddc75c2 --- /dev/null +++ b/e2e/tests/03.14_node_satellite_can_sync_dates_and_times.lux @@ -0,0 +1,88 @@ +[doc NodeJS Satellite correctly syncs DATE and TIME values from and to Electric] +[include _shared.luxinc] +[include _satellite_macros.luxinc] + +[invoke setup] + +[shell pg_1] + [local sql= + """ + CREATE TABLE public.datetimes ( + id TEXT PRIMARY KEY DEFAULT uuid_generate_v4(), + d DATE, + t TIME + ); + CALL electric.electrify('public.datetimes'); + """] + [invoke migrate_pg 20230913 $sql] + +[invoke setup_client 1 electric_1 5133] + +[shell satellite_1] + [invoke node_await_table "datetimes"] + [invoke node_sync_table "datetimes"] + +[shell pg_1] + !INSERT INTO public.datetimes (id, d, t) VALUES ('001', '2023-08-23', '11:00:59'), \ + ('002', '01-01-0001', '00:59:03.11'), \ + ('003', 'Feb 29 6000', '23:05:17.999999'); + ??INSERT 0 3 + +[shell satellite_1] + [invoke node_await_get_from_table "datetimes" "003"] + + ??id: '001' + ??d: '2023-08-23' + ??t: '11:00:59' + + ??id: '002' + ??d: '0001-01-01' + ??t: '00:59:03.11' + + ??id: '003' + ??d: '6000-02-29' + ??t: '23:05:17.999999' + + [invoke node_await_insert_extended_into "datetimes" "{id: '004', d: '1999-12-31'}"] + [invoke node_await_insert_extended_into "datetimes" "{id: '005', t: '00:00:00.000'}"] + +[shell pg_1] + [invoke wait-for "SELECT * FROM public.datetimes;" "005" 10 $psql] + + !SELECT * FROM public.datetimes; + ??001 | 2023-08-23 | 11:00:59 + ??002 | 0001-01-01 | 00:59:03.11 + ??003 | 6000-02-29 | 23:05:17.999999 + ??004 | 1999-12-31 | + ??005 | | 00:00:00 + +# Start a new Satellite client and verify that it receives all dates and times +[invoke setup_client 2 electric_1 5133] + +[shell satellite_2] + [invoke node_await_table "datetimes"] + [invoke node_sync_table "datetimes"] + + [invoke node_await_get_from_table "datetimes" "005"] + ??id: '001' + ??d: '2023-08-23' + ??t: '11:00:59' + + ??id: '002' + ??d: '0001-01-01' + ??t: '00:59:03.11' + + ??id: '003' + ??d: '6000-02-29' + ??t: '23:05:17.999999' + + ??id: '004' + ??d: '1999-12-31' + ??t: null + + ??id: '005' + ??d: null + ??t: '00:00:00' + +[cleanup] + [invoke teardown]