feat(electric): Add support for BOOLEAN data type in electrified tabl…

…es (#413) Closes VAX-821. --------- Co-authored-by: Ilia Borovitinov <ilia@electric-sql.com>
electric-sql · Sep 14, 2023 · cf4ee7c · cf4ee7c
1 parent 33ed7e8
commit cf4ee7c
Show file tree

Hide file tree

Showing 9 changed files with 197 additions and 11 deletions.
diff --git a/.changeset/brown-pots-brake.md b/.changeset/brown-pots-brake.md
@@ -0,0 +1,5 @@
+---
+"@core/electric": patch
+---
+
+Implement support for the BOOLEAN column type in electrified tables
diff --git a/clients/typescript/src/satellite/client.ts b/clients/typescript/src/satellite/client.ts
@@ -994,7 +994,7 @@ export function serializeRow(rec: Record, relation: Relation): SatOpRow {
   const recordValues = relation!.columns.reduce(
     (acc: Uint8Array[], c: RelationColumn) => {
       if (rec[c.name] != null) {
-        acc.push(serializeColumnData(rec[c.name]!))
+        acc.push(serializeColumnData(rec[c.name]!, c.type))
       } else {
         acc.push(serializeNullData())
         setMaskBit(recordNullBitMask, recordNumColumn)
@@ -1054,6 +1054,8 @@ function deserializeColumnData(
     case 'UUID':
     case 'VARCHAR':
       return typeDecoder.text(column)
+    case 'BOOL':
+      return typeDecoder.bool(column)
     case 'FLOAT4':
     case 'FLOAT8':
     case 'INT':
@@ -1070,8 +1072,16 @@ function deserializeColumnData(
 }
 
 // All values serialized as textual representation
-function serializeColumnData(column: string | number): Uint8Array {
-  return typeEncoder.text(column as string)
+function serializeColumnData(
+  col_val: string | number,
+  col_type: string
+): Uint8Array {
+  switch (col_type.toUpperCase()) {
+    case 'BOOL':
+      return typeEncoder.bool(col_val as number)
+    default:
+      return typeEncoder.text(col_val as string)
+  }
 }
 
 function serializeNullData(): Uint8Array {

diff --git a/clients/typescript/src/util/common.ts b/clients/typescript/src/util/common.ts
@@ -18,11 +18,13 @@ setGlobalUUID(
 )
 
 export const typeDecoder = {
+  bool: bytesToBool,
   number: bytesToNumber,
   text: bytesToString,
 }
 
 export const typeEncoder = {
+  bool: boolToBytes,
   number: numberToBytes,
   text: (string: string) => new TextEncoder().encode(string),
 }
@@ -38,6 +40,23 @@ export const base64 = {
 
 export const DEFAULT_LOG_POS = numberToBytes(0)
 
+const trueByte = 't'.charCodeAt(0)
+const falseByte = 'f'.charCodeAt(0)
+
+export function boolToBytes(b: number) {
+  if (b !== 0 && b !== 1) {
+    throw new Error(`Invalid boolean value: ${b}`)
+  }
+  return new Uint8Array([b === 1 ? trueByte : falseByte])
+}
+export function bytesToBool(bs: Uint8Array) {
+  if (bs.length === 1 && (bs[0] === trueByte || bs[0] === falseByte)) {
+    return bs[0] === trueByte ? 1 : 0
+  }
+
+  throw new Error(`Invalid binary-encoded boolean value: ${bs}`)
+}
+
 export function numberToBytes(i: number) {
   return Uint8Array.of(
     (i & 0xff000000) >> 24,

diff --git a/clients/typescript/test/satellite/serialization.test.ts b/clients/typescript/test/satellite/serialization.test.ts
@@ -17,6 +17,9 @@ test('serialize/deserialize row data', async (t) => {
       { name: 'int2', type: 'INTEGER', isNullable: true },
       { name: 'float1', type: 'FLOAT4', isNullable: true },
       { name: 'float2', type: 'FLOAT4', isNullable: true },
+      { name: 'bool1', type: 'BOOL', isNullable: true },
+      { name: 'bool2', type: 'BOOL', isNullable: true },
+      { name: 'bool3', type: 'BOOL', isNullable: true },
     ],
   }
 
@@ -28,6 +31,9 @@ test('serialize/deserialize row data', async (t) => {
     int2: -30,
     float1: 1.1,
     float2: -30.3,
+    bool1: 1,
+    bool2: 0,
+    bool3: null,
   }
   const s_row = serializeRow(record, rel)
   const d_row = deserializeRow(s_row, rel)

diff --git a/components/electric/lib/electric/satellite/serialization.ex b/components/electric/lib/electric/satellite/serialization.ex
@@ -24,6 +24,7 @@ defmodule Electric.Satellite.Serialization do
   @spec supported_pg_types :: [atom]
   def supported_pg_types do
     ~w[
+      bool
       date
       float8
       int2 int4 int8
@@ -478,6 +479,11 @@ defmodule Electric.Satellite.Serialization do
   """
   @spec decode_column_value!(binary, atom) :: binary
 
+  def decode_column_value!(val, :bool) do
+    true = val in ["t", "f"]
+    val
+  end
+
   def decode_column_value!(val, type) when type in [:bytea, :text, :varchar] do
     val
   end

diff --git a/components/electric/test/electric/postgres/extension_test.exs b/components/electric/test/electric/postgres/extension_test.exs
@@ -441,7 +441,8 @@ defmodule Electric.Postgres.ExtensionTest do
                  ts TIMESTAMP,
                  tstz TIMESTAMPTZ,
                  d DATE,
-                 t TIME
+                 t TIME,
+                 flag BOOLEAN
                );
                CALL electric.electrify('public.t1');
                """)

diff --git a/components/electric/test/electric/satellite/serialization_test.exs b/components/electric/test/electric/satellite/serialization_test.exs
@@ -20,7 +20,8 @@ defmodule Electric.Satellite.SerializationTest do
         "real" => "-3.14",
         "id" => uuid,
         "date" => "2024-12-24",
-        "time" => "12:01:00.123"
+        "time" => "12:01:00.123",
+        "bool" => "t"
       }
 
       columns = [
@@ -32,11 +33,23 @@ defmodule Electric.Satellite.SerializationTest do
         %{name: "var", type: :varchar},
         %{name: "real", type: :float8},
         %{name: "date", type: :date},
-        %{name: "time", type: :time}
+        %{name: "time", type: :time},
+        %{name: "bool", type: :bool}
       ]
 
       assert %SatOpRow{
-               values: ["", "", "4", uuid, "13", "...", "-3.14", "2024-12-24", "12:01:00.123"],
+               values: [
+                 "",
+                 "",
+                 "4",
+                 uuid,
+                 "13",
+                 "...",
+                 "-3.14",
+                 "2024-12-24",
+                 "12:01:00.123",
+                 "t"
+               ],
                nulls_bitmask: <<0b11000000, 0>>
              } == Serialization.map_to_row(data, columns)
     end
@@ -79,7 +92,8 @@ defmodule Electric.Satellite.SerializationTest do
           "2023-08-15 17:20:31Z",
           "",
           "0400-02-29",
-          "03:59:59"
+          "03:59:59",
+          "f"
         ]
       }
 
@@ -93,7 +107,8 @@ defmodule Electric.Satellite.SerializationTest do
         %{name: "tz", type: :timestamptz},
         %{name: "x", type: :float4, nullable?: true},
         %{name: "date", type: :date},
-        %{name: "time", type: :time}
+        %{name: "time", type: :time},
+        %{name: "bool", type: :bool}
       ]
 
       assert %{
@@ -106,7 +121,8 @@ defmodule Electric.Satellite.SerializationTest do
                "tz" => "2023-08-15 17:20:31Z",
                "x" => nil,
                "date" => "0400-02-29",
-               "time" => "03:59:59"
+               "time" => "03:59:59",
+               "bool" => "f"
              } == Serialization.decode_record!(row, columns)
     end
 
@@ -147,7 +163,12 @@ defmodule Electric.Satellite.SerializationTest do
         {"010203", :time},
         {"016003", :time},
         {"00:00:00.", :time},
-        {"00:00:00.1234567", :time}
+        {"00:00:00.1234567", :time},
+        {"true", :bool},
+        {"false", :bool},
+        {"yes", :bool},
+        {"no", :bool},
+        {"-1", :bool}
       ]
 
       Enum.each(test_data, fn {val, type} ->

diff --git a/components/electric/test/electric/satellite/ws_validations_test.exs b/components/electric/test/electric/satellite/ws_validations_test.exs
@@ -84,6 +84,52 @@ defmodule Electric.Satellite.WsValidationsTest do
     end)
   end
 
+  test "validates boolean values", ctx do
+    vsn = "2023072502"
+
+    :ok =
+      migrate(
+        ctx.db,
+        vsn,
+        "public.foo",
+        "CREATE TABLE public.foo (id TEXT PRIMARY KEY, b BOOLEAN)"
+      )
+
+    valid_records = [
+      %{"id" => "1", "b" => "t"},
+      %{"id" => "2", "b" => "f"},
+      %{"id" => "3", "b" => ""}
+    ]
+
+    within_replication_context(ctx, vsn, fn conn ->
+      Enum.each(valid_records, fn record ->
+        tx_op_log = serialize_trans(record)
+        MockClient.send_data(conn, tx_op_log)
+      end)
+
+      refute_received {^conn, %SatErrorResp{error_type: :INVALID_REQUEST}}
+    end)
+
+    invalid_records = [
+      %{"id" => "10", "b" => "1"},
+      %{"id" => "11", "b" => "0"},
+      %{"id" => "12", "b" => "True"},
+      %{"id" => "13", "b" => "false"},
+      %{"id" => "14", "b" => "+"},
+      %{"id" => "15", "b" => "-"},
+      %{"id" => "16", "b" => "yes"},
+      %{"id" => "17", "b" => "no"}
+    ]
+
+    Enum.each(invalid_records, fn record ->
+      within_replication_context(ctx, vsn, fn conn ->
+        tx_op_log = serialize_trans(record)
+        MockClient.send_data(conn, tx_op_log)
+        assert_receive {^conn, %SatErrorResp{error_type: :INVALID_REQUEST}}, @receive_timeout
+      end)
+    end)
+  end
+
   test "validates integer values", ctx do
     vsn = "2023072502"
 

diff --git a/e2e/tests/03.15_node_satellite_can_sync_booleans.lux b/e2e/tests/03.15_node_satellite_can_sync_booleans.lux
@@ -0,0 +1,72 @@
+[doc NodeJS Satellite correctly syncs BOOL values from and to Electric]
+[include _shared.luxinc]
+[include _satellite_macros.luxinc]
+
+[invoke setup]
+
+[shell pg_1]
+    [local sql=
+        """
+        CREATE TABLE public.bools (
+            id TEXT PRIMARY KEY DEFAULT uuid_generate_v4(),
+            b BOOLEAN
+        );
+        CALL electric.electrify('public.bools');
+        """]
+    [invoke migrate_pg 20230908 $sql]
+
+[invoke setup_client 1 electric_1 5133]
+
+[shell satellite_1]
+    [invoke node_await_table "bools"]
+    [invoke node_sync_table "bools"]
+
+[shell pg_1]
+    !INSERT INTO public.bools (id, b) VALUES ('001', true), ('002', false), ('003', NULL);
+    ??INSERT 0 3
+
+[shell satellite_1]
+    [invoke node_await_get_from_table "bools" "003"]
+
+    ??id: '001'
+    ??b: 1
+
+    ??id: '002'
+    ??b: 0
+
+    ??id: '003'
+    ??b: null
+
+    [invoke node_await_insert_extended_into "bools" "{id: '004', b: 1}"]
+
+[shell pg_1]
+    [invoke wait-for "SELECT * FROM public.bools;" "004" 10 $psql]
+
+    !SELECT * FROM public.bools;
+    ??001 | t
+    ??002 | f
+    ??003 | <NULL>
+    ??004 | t
+
+# Start a new Satellite client and verify that it receives all rows
+[invoke setup_client 2 electric_1 5133]
+
+[shell satellite_2]
+    [invoke node_await_table "bools"]
+    [invoke node_sync_table "bools"]
+
+    [invoke node_await_get_from_table "bools" "004"]
+    ??id: '001'
+    ??b: 1
+
+    ??id: '002'
+    ??b: 0
+
+    ??id: '003'
+    ??b: null
+
+    ??id: '004'
+    ??b: 1
+
+[cleanup]
+  [invoke teardown]