chore(electric): Clean up templated SQL routines

components/electric/Dockerfile

@@ -24,9 +24,10 @@ COPY mix.* /app/
 RUN mix deps.get
 RUN mix deps.compile
 
+COPY config/*runtime.exs /app/config/
 COPY lib /app/lib/
+COPY priv /app/priv
 COPY src /app/src/
-COPY config/*runtime.exs /app/config/
 
 ARG ELECTRIC_VERSION=local
 ARG MAKE_RELEASE_TASK=release

components/electric/lib/electric/postgres/extension.ex

@@ -287,14 +287,14 @@ defmodule Electric.Postgres.Extension do
 
   @spec define_functions(conn) :: :ok
   def define_functions(conn) do
-    Enum.each(Functions.list(), fn {name, sql} ->
-      case :epgsql.squery(conn, sql) do
-        {:ok, [], []} ->
-          Logger.debug("Successfully (re)defined SQL function/procedure '#{name}'")
-          :ok
-
-        error ->
-          raise "Failed to define function '#{name}' with error: #{inspect(error)}"
+    Enum.each(Functions.list(), fn {path, sql} ->
+      conn
+      |> :epgsql.squery(sql)
+      |> List.wrap()
+      |> Enum.find(&(not match?({:ok, [], []}, &1)))
+      |> case do
+        nil -> Logger.debug("Successfully (re)defined SQL routine from '#{path}'")
+        error -> raise "Failed to define SQL routine from '#{path}' with error: #{inspect(error)}"
       end
     end)
   end
@@ -357,7 +357,9 @@ defmodule Electric.Postgres.Extension do
           end)
         end)
 
-      :ok = define_functions(txconn)
+      if module == __MODULE__ do
+        :ok = define_functions(txconn)
+      end
 
       {:ok, newly_applied_versions}
     end)

components/electric/lib/electric/postgres/extension/functions.ex

@@ -3,24 +3,26 @@ defmodule Electric.Postgres.Extension.Functions do
   This module organizes SQL functions that are to be defined in Electric's internal database schema.
   """
 
-  alias Electric.Postgres.Extension
+  # Import all functions from the Extension module to make them available for calling inside SQL function templates when
+  # those templates are being evaludated by EEx.
+  import Electric.Postgres.Extension, warn: false
+
   require EEx
 
-  sql_files =
-    "functions/*.sql.eex"
-    |> Path.expand(__DIR__)
-    |> Path.wildcard()
+  @template_dir "priv/sql_function_templates"
 
-  function_names =
-    for path <- sql_files do
-      @external_resource path
+  template_dir_path = Application.app_dir(:electric, @template_dir)
+  sql_template_paths = Path.wildcard(template_dir_path <> "/**/*.sql.eex")
 
+  function_paths =
+    for path <- sql_template_paths do
+      relpath = Path.relative_to(path, template_dir_path)
       name = path |> Path.basename(".sql.eex") |> String.to_atom()
-      _ = EEx.function_from_file(:def, name, path, [:assigns])
-
-      name
+      {relpath, name}
     end
 
+  function_names = for {_relpath, name} <- function_paths, do: name
+
   fn_name_type =
     Enum.reduce(function_names, fn name, code ->
       quote do
@@ -29,22 +31,23 @@ defmodule Electric.Postgres.Extension.Functions do
     end)
 
   @typep name :: unquote(fn_name_type)
-  @typep sql :: String.t()
-  @type function_list :: [{name, sql}]
+  @typep sql :: binary
+  @type function_list :: [{Path.t(), sql}]
 
+  @function_paths function_paths
   @function_names function_names
 
   @doc """
-  Get a list of `{name, SQL}` pairs where the the SQL code contains the definition of a function (or multiple functions).
+  Get a list of `{name, SQL}` pairs where the SQL code contains the definition of a function (or multiple functions).
 
   Every function in the list is defined as `CREATE OR REPLACE FUNCTION`.
   """
   # NOTE(alco): Eventually, we're hoping to move all function definitions out of migrations and define them all
   # here. See VAX-1016 for details.
   @spec list :: function_list
   def list do
-    for name <- @function_names do
-      {name, by_name(name)}
+    for {relpath, _name} <- @function_paths do
+      {relpath, eval_template(relpath)}
     end
   end
 
@@ -56,19 +59,18 @@ defmodule Electric.Postgres.Extension.Functions do
   """
   @spec by_name(name) :: sql
   def by_name(name) when name in @function_names do
-    apply(__MODULE__, name, [assigns()])
+    {relpath, ^name} = List.keyfind(@function_paths, name, 1)
+    eval_template(relpath)
   end
 
-  # This map of assigns is the same for all function templates.
-  defp assigns do
-    %{
-      schema: Extension.schema(),
-      ddl_table: Extension.ddl_table(),
-      txid_type: Extension.txid_type(),
-      txts_type: Extension.txts_type(),
-      version_table: Extension.version_table(),
-      electrified_tracking_table: Extension.electrified_tracking_table(),
-      publication_name: Extension.publication_name()
-    }
+  defp eval_template(relpath) do
+    # This hack is necessary to get a meaningful error when EEx fails to evaluate the template.
+    # Without it, errors will say that it is lib/electric/postgres/extension/functions.ex that failed to compile.
+    env = %{__ENV__ | file: relpath}
+
+    Path.join(Application.app_dir(:electric, @template_dir), relpath)
+    |> EEx.compile_file(file: relpath)
+    |> Code.eval_quoted([], env)
+    |> elem(0)
   end
 end

components/electric/lib/electric/postgres/extension/migrations/20230328113927_setup_extension.ex

@@ -52,7 +52,7 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230328113927 do
       """,
       ##################
       """
-      CREATE PUBLICATION "#{publication_name}"; 
+      CREATE PUBLICATION "#{publication_name}";
       """,
       Extension.add_table_to_publication_sql(ddl_table)
     ]

components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers.ex

@@ -19,11 +19,6 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230512000000_confli
   def up(schema) do
     [
       @contents["electric_tag_type_and_operators"],
-      @contents["utility_functions"],
-      # This function definition is included here because it is referenced in the definition of
-      # "trigger_function_installers" below it.
-      Extension.Functions.by_name(:perform_reordered_op_installer_function),
-      Extension.Functions.by_name(:__session_replication_role),
       @contents["trigger_function_installers"],
       @contents["shadow_table_creation_and_update"]
       # We need to actually run shadow table creation/updates, but that's handled in the next migration.

components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function.ex

@@ -1,14 +1,8 @@
 defmodule Electric.Postgres.Extension.Migrations.Migration_20230605141256_ElectrifyFunction do
   alias Electric.Postgres.Extension
 
-  require EEx
-
   @behaviour Extension.Migration
 
-  sql_template = Path.expand("20230605141256_electrify_function/electrify.sql.eex", __DIR__)
-
-  @external_resource sql_template
-
   @impl true
   def version, do: 2023_06_05_14_12_56
 
@@ -25,12 +19,10 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230605141256_Electr
           oid         oid NOT NULL,
           created_at  timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP,
           CONSTRAINT unique_table_name UNIQUE (schema_name, table_name)
-      );
-      """,
-      """
-      CREATE INDEX electrified_tracking_table_name_idx ON #{electrified_tracking_table} (schema_name, table_name);
-      CREATE INDEX electrified_tracking_table_name_oid ON #{electrified_tracking_table} (oid);
+      )
       """,
+      "CREATE INDEX electrified_tracking_table_name_idx ON #{electrified_tracking_table} (schema_name, table_name)",
+      "CREATE INDEX electrified_tracking_table_name_oid ON #{electrified_tracking_table} (oid)",
       Extension.add_table_to_publication_sql(electrified_tracking_table)
     ]
   end