feat(electric): Add support for TIME data type in electrified tables (#…

…427)

Closes VAX-857.

.changeset/shy-days-whisper.md

@@ -0,0 +1,5 @@
+---
+"@core/electric": patch
+---
+
+Implement support for the TIME column type in electrified tables.

clients/typescript/src/satellite/client.ts

@@ -1048,6 +1048,7 @@ function deserializeColumnData(
     case 'CHAR':
     case 'DATE':
     case 'TEXT':
+    case 'TIME':
     case 'TIMESTAMP':
     case 'TIMESTAMPTZ':
     case 'UUID':

components/electric/lib/electric/satellite/serialization.ex

@@ -28,6 +28,7 @@ defmodule Electric.Satellite.Serialization do
       float8
       int2 int4 int8
       text
+      time
       timestamp timestamptz
       uuid
       varchar
@@ -512,6 +513,25 @@ defmodule Electric.Satellite.Serialization do
     val
   end
 
+  def decode_column_value!(val, :time) do
+    <<hh::binary-2, ?:, mm::binary-2, ?:, ss::binary-2>> <> frac = val
+
+    hours = String.to_integer(hh)
+    true = hours in 0..23
+
+    minutes = String.to_integer(mm)
+    true = minutes in 0..59
+
+    seconds = String.to_integer(ss)
+    true = seconds in 0..59
+
+    :ok = validate_fractional_seconds(frac)
+
+    _ = Time.from_iso8601!(val)
+
+    val
+  end
+
   def decode_column_value!(val, :timestamp) do
     # NaiveDateTime silently discards time zone offset if it is present in the string. But we want to reject such strings
     # because values of type `timestamp` must not have an offset.
@@ -559,4 +579,13 @@ defmodule Electric.Satellite.Serialization do
   #   [1]: https://www.postgresql.org/docs/current/datatype-datetime.html
   #   [2]: https://www.sqlite.org/lang_datefunc.html
   defp assert_year_in_range(year) when year in 1..9999, do: :ok
+
+  defp validate_fractional_seconds(""), do: :ok
+
+  defp validate_fractional_seconds("." <> fs_str) do
+    # Fractional seconds must not exceed 6 decimal digits, otherwise Postgres will round the last digit up or down.
+    true = byte_size(fs_str) <= 6
+    _ = String.to_integer(fs_str)
+    :ok
+  end
 end

components/electric/test/electric/postgres/extension_test.exs

@@ -440,7 +440,8 @@ defmodule Electric.Postgres.ExtensionTest do
                  real8b DOUBLE PRECISION,
                  ts TIMESTAMP,
                  tstz TIMESTAMPTZ,
-                 d DATE
+                 d DATE,
+                 t TIME
                );
                CALL electric.electrify('public.t1');
                """)

components/electric/test/electric/satellite/serialization_test.exs

@@ -19,7 +19,8 @@ defmodule Electric.Satellite.SerializationTest do
         "var" => "...",
         "real" => "-3.14",
         "id" => uuid,
-        "date" => "2024-12-24"
+        "date" => "2024-12-24",
+        "time" => "12:01:00.123"
       }
 
       columns = [
@@ -30,12 +31,13 @@ defmodule Electric.Satellite.SerializationTest do
         %{name: "int", type: :int4},
         %{name: "var", type: :varchar},
         %{name: "real", type: :float8},
-        %{name: "date", type: :date}
+        %{name: "date", type: :date},
+        %{name: "time", type: :time}
       ]
 
       assert %SatOpRow{
-               values: ["", "", "4", uuid, "13", "...", "-3.14", "2024-12-24"],
-               nulls_bitmask: <<0b11000000>>
+               values: ["", "", "4", uuid, "13", "...", "-3.14", "2024-12-24", "12:01:00.123"],
+               nulls_bitmask: <<0b11000000, 0>>
              } == Serialization.map_to_row(data, columns)
     end
 
@@ -76,7 +78,8 @@ defmodule Electric.Satellite.SerializationTest do
           "2023-08-15 17:20:31",
           "2023-08-15 17:20:31Z",
           "",
-          "0400-02-29"
+          "0400-02-29",
+          "03:59:59"
         ]
       }
 
@@ -89,7 +92,8 @@ defmodule Electric.Satellite.SerializationTest do
         %{name: "t", type: :timestamp},
         %{name: "tz", type: :timestamptz},
         %{name: "x", type: :float4, nullable?: true},
-        %{name: "date", type: :date}
+        %{name: "date", type: :date},
+        %{name: "time", type: :time}
       ]
 
       assert %{
@@ -101,7 +105,8 @@ defmodule Electric.Satellite.SerializationTest do
                "t" => "2023-08-15 17:20:31",
                "tz" => "2023-08-15 17:20:31Z",
                "x" => nil,
-               "date" => "0400-02-29"
+               "date" => "0400-02-29",
+               "time" => "03:59:59"
              } == Serialization.decode_record!(row, columns)
     end
 
@@ -132,7 +137,17 @@ defmodule Electric.Satellite.SerializationTest do
         {"1999-31-12", :date},
         {"20230815", :date},
         {"-2023-08-15", :date},
-        {"12-12-12", :date}
+        {"12-12-12", :date},
+        {"24:00:00", :time},
+        {"-12:00:00", :time},
+        {"22:01", :time},
+        {"02:60:00", :time},
+        {"02:00:60", :time},
+        {"1:2:3", :time},
+        {"010203", :time},
+        {"016003", :time},
+        {"00:00:00.", :time},
+        {"00:00:00.1234567", :time}
       ]
 
       Enum.each(test_data, fn {val, type} ->

components/electric/test/electric/satellite/ws_validations_test.exs

@@ -266,6 +266,51 @@ defmodule Electric.Satellite.WsValidationsTest do
     end)
   end
 
+  test "validates time values", ctx do
+    vsn = "2023091101"
+
+    :ok =
+      migrate(ctx.db, vsn, "public.foo", "CREATE TABLE public.foo (id TEXT PRIMARY KEY, t time)")
+
+    valid_records = [
+      %{"id" => "1", "t" => "00:00:00"},
+      %{"id" => "2", "t" => "23:59:59"},
+      %{"id" => "3", "t" => "00:00:00.332211"},
+      %{"id" => "4", "t" => "11:11:11.11"}
+    ]
+
+    within_replication_context(ctx, vsn, fn conn ->
+      Enum.each(valid_records, fn record ->
+        tx_op_log = serialize_trans(record)
+        MockClient.send_data(conn, tx_op_log)
+      end)
+    end)
+
+    refute_receive {_, %SatErrorResp{error_type: :INVALID_REQUEST}}, @receive_timeout
+
+    invalid_records = [
+      %{"id" => "10", "t" => "now"},
+      %{"id" => "11", "t" => "::"},
+      %{"id" => "12", "t" => "20:12"},
+      %{"id" => "13", "t" => "T18:00"},
+      %{"id" => "14", "t" => "l2:o6:t0"},
+      %{"id" => "15", "t" => "1:20:23"},
+      %{"id" => "16", "t" => "02:02:03-08:00"},
+      %{"id" => "17", "t" => "01:00:00+0"},
+      %{"id" => "18", "t" => "99:99:99"},
+      %{"id" => "19", "t" => "12:1:0"},
+      %{"id" => "20", "t" => ""}
+    ]
+
+    Enum.each(invalid_records, fn record ->
+      within_replication_context(ctx, vsn, fn conn ->
+        tx_op_log = serialize_trans(record)
+        MockClient.send_data(conn, tx_op_log)
+        assert_receive {^conn, %SatErrorResp{error_type: :INVALID_REQUEST}}, @receive_timeout
+      end)
+    end)
+  end
+
   test "validates timestamp values", ctx do
     vsn = "2023072505"
 

e2e/tests/03.14_node_satellite_can_sync_dates_and_times.lux

@@ -0,0 +1,88 @@
+[doc NodeJS Satellite correctly syncs DATE and TIME values from and to Electric]
+[include _shared.luxinc]
+[include _satellite_macros.luxinc]
+
+[invoke setup]
+
+[shell pg_1]
+    [local sql=
+        """
+        CREATE TABLE public.datetimes (
+            id TEXT PRIMARY KEY DEFAULT uuid_generate_v4(),
+            d DATE,
+            t TIME
+        );
+        CALL electric.electrify('public.datetimes');
+        """]
+    [invoke migrate_pg 20230913 $sql]
+
+[invoke setup_client 1 electric_1 5133]
+
+[shell satellite_1]
+    [invoke node_await_table "datetimes"]
+    [invoke node_sync_table "datetimes"]
+
+[shell pg_1]
+    !INSERT INTO public.datetimes (id, d, t) VALUES ('001', '2023-08-23', '11:00:59'), \
+                                                    ('002', '01-01-0001', '00:59:03.11'), \
+                                                    ('003', 'Feb 29 6000', '23:05:17.999999');
+    ??INSERT 0 3
+
+[shell satellite_1]
+    [invoke node_await_get_from_table "datetimes" "003"]
+
+    ??id: '001'
+    ??d: '2023-08-23'
+    ??t: '11:00:59'
+
+    ??id: '002'
+    ??d: '0001-01-01'
+    ??t: '00:59:03.11'
+
+    ??id: '003'
+    ??d: '6000-02-29'
+    ??t: '23:05:17.999999'
+
+    [invoke node_await_insert_extended_into "datetimes" "{id: '004', d: '1999-12-31'}"]
+    [invoke node_await_insert_extended_into "datetimes" "{id: '005', t: '00:00:00.000'}"]
+
+[shell pg_1]
+    [invoke wait-for "SELECT * FROM public.datetimes;" "005" 10 $psql]
+
+    !SELECT * FROM public.datetimes;
+    ??001 | 2023-08-23 | 11:00:59
+    ??002 | 0001-01-01 | 00:59:03.11
+    ??003 | 6000-02-29 | 23:05:17.999999
+    ??004 | 1999-12-31 | <NULL>
+    ??005 | <NULL>     | 00:00:00
+
+# Start a new Satellite client and verify that it receives all dates and times
+[invoke setup_client 2 electric_1 5133]
+
+[shell satellite_2]
+    [invoke node_await_table "datetimes"]
+    [invoke node_sync_table "datetimes"]
+
+    [invoke node_await_get_from_table "datetimes" "005"]
+    ??id: '001'
+    ??d: '2023-08-23'
+    ??t: '11:00:59'
+
+    ??id: '002'
+    ??d: '0001-01-01'
+    ??t: '00:59:03.11'
+
+    ??id: '003'
+    ??d: '6000-02-29'
+    ??t: '23:05:17.999999'
+
+    ??id: '004'
+    ??d: '1999-12-31'
+    ??t: null
+
+    ??id: '005'
+    ??d: null
+    ??t: '00:00:00'
+
+[cleanup]
+  [invoke teardown]