From 126dff4e4b5c5d071fe3cb7197e356b4753a6794 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 29 Sep 2023 17:56:24 +0300 Subject: [PATCH 1/9] Remove leftover SQL function/procedure definitions from old migrations The functions/procedures are now defined at run time. --- .../lib/electric/postgres/extension.ex | 14 +- .../upsert_acknowledged_client_lsn.sql.eex} | 10 +- .../20230328113927_setup_extension.ex | 2 +- ...0512000000_conflict_resolution_triggers.ex | 5 - .../20230605141256_electrify_function.ex | 14 +- .../electrify.sql.eex | 300 ------------------ ...29000000_acknowledged_client_lsns_table.ex | 22 +- .../electric/test/support/extension_case.ex | 1 + 8 files changed, 23 insertions(+), 345 deletions(-) rename components/electric/lib/electric/postgres/extension/{migrations/20230829000000_acknowledged_client_lsns_table/deduplicating_trigger.sql => functions/upsert_acknowledged_client_lsn.sql.eex} (62%) delete mode 100644 components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function/electrify.sql.eex diff --git a/components/electric/lib/electric/postgres/extension.ex b/components/electric/lib/electric/postgres/extension.ex index 347990d2c5..5bcef8ce1d 100644 --- a/components/electric/lib/electric/postgres/extension.ex +++ b/components/electric/lib/electric/postgres/extension.ex @@ -288,13 +288,13 @@ defmodule Electric.Postgres.Extension do @spec define_functions(conn) :: :ok def define_functions(conn) do Enum.each(Functions.list(), fn {name, sql} -> - case :epgsql.squery(conn, sql) do - {:ok, [], []} -> - Logger.debug("Successfully (re)defined SQL function/procedure '#{name}'") - :ok - - error -> - raise "Failed to define function '#{name}' with error: #{inspect(error)}" + conn + |> :epgsql.squery(sql) + |> List.wrap() + |> Enum.find(&(not match?({:ok, [], []}, &1))) + |> case do + nil -> Logger.debug("Successfully (re)defined SQL function/procedure '#{name}'") + error -> raise "Failed to define function '#{name}' with error: #{inspect(error)}" end end) end diff --git a/components/electric/lib/electric/postgres/extension/migrations/20230829000000_acknowledged_client_lsns_table/deduplicating_trigger.sql b/components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex similarity index 62% rename from components/electric/lib/electric/postgres/extension/migrations/20230829000000_acknowledged_client_lsns_table/deduplicating_trigger.sql rename to components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex index 4b5d2bb42c..083b427223 100644 --- a/components/electric/lib/electric/postgres/extension/migrations/20230829000000_acknowledged_client_lsns_table/deduplicating_trigger.sql +++ b/components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex @@ -10,18 +10,10 @@ BEGIN INSERT INTO electric.acknowledged_client_lsns AS t VALUES (NEW.client_id, NEW.lsn) - ON CONFLICT (client_id) DO UPDATE + ON CONFLICT (client_id) DO UPDATE SET lsn = NEW.lsn WHERE t.lsn IS DISTINCT FROM NEW.lsn; RETURN NULL; END; $function$; - -CREATE OR REPLACE TRIGGER upsert_acknowledged_client_lsn -BEFORE INSERT ON electric.acknowledged_client_lsns -FOR EACH ROW -WHEN (pg_trigger_depth() < 1) -EXECUTE FUNCTION electric.upsert_acknowledged_client_lsn(); - -ALTER TABLE electric.acknowledged_client_lsns ENABLE REPLICA TRIGGER upsert_acknowledged_client_lsn; diff --git a/components/electric/lib/electric/postgres/extension/migrations/20230328113927_setup_extension.ex b/components/electric/lib/electric/postgres/extension/migrations/20230328113927_setup_extension.ex index f92bbc4f10..2761af4f9e 100644 --- a/components/electric/lib/electric/postgres/extension/migrations/20230328113927_setup_extension.ex +++ b/components/electric/lib/electric/postgres/extension/migrations/20230328113927_setup_extension.ex @@ -52,7 +52,7 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230328113927 do """, ################## """ - CREATE PUBLICATION "#{publication_name}"; + CREATE PUBLICATION "#{publication_name}"; """, Extension.add_table_to_publication_sql(ddl_table) ] diff --git a/components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers.ex b/components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers.ex index a799f5f8bc..8a7b74c6a4 100644 --- a/components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers.ex +++ b/components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers.ex @@ -19,11 +19,6 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230512000000_confli def up(schema) do [ @contents["electric_tag_type_and_operators"], - @contents["utility_functions"], - # This function definition is included here because it is referenced in the definition of - # "trigger_function_installers" below it. - Extension.Functions.by_name(:perform_reordered_op_installer_function), - Extension.Functions.by_name(:__session_replication_role), @contents["trigger_function_installers"], @contents["shadow_table_creation_and_update"] # We need to actually run shadow table creation/updates, but that's handled in the next migration. diff --git a/components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function.ex b/components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function.ex index d8481868a3..11962f1977 100644 --- a/components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function.ex +++ b/components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function.ex @@ -1,14 +1,8 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230605141256_ElectrifyFunction do alias Electric.Postgres.Extension - require EEx - @behaviour Extension.Migration - sql_template = Path.expand("20230605141256_electrify_function/electrify.sql.eex", __DIR__) - - @external_resource sql_template - @impl true def version, do: 2023_06_05_14_12_56 @@ -25,12 +19,10 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230605141256_Electr oid oid NOT NULL, created_at timestamp with time zone NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT unique_table_name UNIQUE (schema_name, table_name) - ); - """, - """ - CREATE INDEX electrified_tracking_table_name_idx ON #{electrified_tracking_table} (schema_name, table_name); - CREATE INDEX electrified_tracking_table_name_oid ON #{electrified_tracking_table} (oid); + ) """, + "CREATE INDEX electrified_tracking_table_name_idx ON #{electrified_tracking_table} (schema_name, table_name)", + "CREATE INDEX electrified_tracking_table_name_oid ON #{electrified_tracking_table} (oid)", Extension.add_table_to_publication_sql(electrified_tracking_table) ] end diff --git a/components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function/electrify.sql.eex b/components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function/electrify.sql.eex deleted file mode 100644 index 67786eb413..0000000000 --- a/components/electric/lib/electric/postgres/extension/migrations/20230605141256_electrify_function/electrify.sql.eex +++ /dev/null @@ -1,300 +0,0 @@ --- vim: set shiftwidth=4:tabstop=4 --- --- you can call electrify using one of these variants: --- --- 1. `CALL electric.electrify('my_table')` --- 2. `CALL electric.electrify('my_schema', 'my_table')` --- 3. `CALL electric.electrify('my_schema.my_table')` --- --- the first two formats also support special characters in the table/schema name: --- --- 4. `CALL electric.electrify('My Schema', 'My Table')` - -CREATE OR REPLACE FUNCTION <%= schema %>.__pg_version() RETURNS integer AS $function$ - SELECT setting::int FROM pg_settings WHERE name = 'server_version_num' -$function$ LANGUAGE SQL; - -------------------------------------------------- - -CREATE OR REPLACE FUNCTION <%= schema %>.__table_schema(table_oid oid) RETURNS name AS $function$ -DECLARE - _schema name; -BEGIN - SELECT pn.nspname INTO _schema - FROM pg_class pc - INNER JOIN pg_namespace pn ON pc.relnamespace = pn.oid - WHERE pc.oid = table_oid; - RETURN _schema; -END; -$function$ LANGUAGE PLPGSQL; - -------------------------------------------------- - -CREATE OR REPLACE FUNCTION <%= schema %>.create_active_migration( - _txid xid8; - _txts timestamptz, - _version text, - _query text DEFAULT NULL -) RETURNS int8 AS -$function$ -DECLARE - trid int8; -BEGIN - IF _query IS NULL THEN - _query := current_query(); - END IF; - RAISE NOTICE 'capture migration: % => %', _version, _query; - INSERT INTO "<%= schema %>"."ddl_commands" (txid, txts, version, query) VALUES - (_txid, _txts, _version, _query) - ON CONFLICT ON CONSTRAINT ddl_table_unique_migrations DO NOTHING - RETURNING id INTO trid; - RETURN trid; -END; -$function$ -LANGUAGE PLPGSQL; - -------------------------------------------------- - -CREATE OR REPLACE FUNCTION <%= schema %>.capture_ddl(query text DEFAULT NULL) RETURNS int8 AS $function$ -DECLARE - _txid xid8; - _txts timestamptz; - _version text; - _trid int8; -BEGIN - SELECT v.txid, v.txts, v.version - INTO _txid, _txts, _version - FROM <%= schema %>.current_migration_version() v; - - _trid := (SELECT <%= schema %>.create_active_migration(_txid, _txts, _version, query)); - - RETURN _trid; -END; -$function$ LANGUAGE PLPGSQL; - -------------------------------------------------- - -CREATE OR REPLACE FUNCTION <%= schema %>.__resolve_table_from_names( - name1 text, - name2 text, - OUT schema_name name, - OUT table_name name, - OUT table_oid regclass -) AS $function$ -DECLARE - _ident text[]; -BEGIN - IF name1 IS NULL AND name2 IS NULL THEN - RAISE EXCEPTION 'no valid table name given'; - ELSIF name2 IS NULL THEN - -- handle table passed as 'schema.table' - IF strpos(name1, '.') > 0 THEN - _ident := parse_ident(name1); - IF array_length(_ident, 1) = 1 THEN - table_name := _ident[1]; - table_oid := (SELECT quote_ident(table_name)::regclass); - schema_name := <%= schema %>.__table_schema(table_oid); - ELSIF array_length(_ident, 1) = 2 THEN - schema_name := _ident[1]; - table_name := _ident[2]; - ELSE - RAISE EXCEPTION 'invalid table name given %', name1; - END IF; - ELSE - table_name := name1; - table_oid := (SELECT quote_ident(table_name)::regclass); - schema_name := <%= schema %>.__table_schema(table_oid); - END IF; - ELSIF name1 IS NOT NULL AND name2 IS NOT NULL THEN - table_name := name2; - schema_name := name1; - ELSE - RAISE EXCEPTION 'no valid table name given'; - END IF; -END; -$function$ LANGUAGE PLPGSQL STABLE; - -------------------------------------------------- - -CREATE OR REPLACE PROCEDURE <%= schema %>.electrify( - name1 text, - name2 text DEFAULT NULL -) AS $function$ -DECLARE - _schema name; - _table text; - _quoted_name text; - _oid regclass; - _create_sql text; -BEGIN - SELECT - table_name, schema_name, table_oid - INTO - _table, _schema, _oid - FROM <%= schema %>.__resolve_table_from_names(name1, name2); - - _quoted_name := format('%I.%I', _schema, _table); - _oid := (SELECT _quoted_name::regclass); - - RAISE NOTICE 'Electrify table %', _quoted_name; - - IF NOT EXISTS (SELECT pc.oid FROM pg_class pc WHERE pc.oid = _oid AND pc.relkind = 'r') THEN - RAISE EXCEPTION '% is not an ordinary table', _quoted_name; - END IF; - - PERFORM <%= schema %>.__validate_table_column_types(_quoted_name); - - EXECUTE format('ALTER TABLE %I.%I REPLICA IDENTITY FULL;', _schema, _table); - - INSERT INTO <%= electrified_tracking_table %> (schema_name, table_name, oid) - VALUES (_schema, _table, _oid) - ON CONFLICT ON CONSTRAINT unique_table_name - DO NOTHING; - - -- insert the required ddl into the migrations table - SELECT <%= schema %>.ddlx_create(_oid) INTO _create_sql; - - RAISE DEBUG '%', _create_sql; - - PERFORM <%= schema %>.capture_ddl(_create_sql); - - IF NOT EXISTS ( - SELECT pr.oid FROM pg_publication_rel pr - INNER JOIN pg_publication pp ON pr.prpubid = pp.oid - WHERE pp.pubname = '<%= publication_name %>' AND pr.prrelid = _oid - ) THEN - EXECUTE format('<%= publication_sql %>;', _schema, _table); - - -- We want to disable any possible hooks from `CREATE TABLE` statements - PERFORM set_config('<%= schema %>.is_in_event_trigger', 'true', true); - PERFORM <%= schema %>.ddlx_make_or_update_shadow_tables('CREATE TABLE', _schema, _oid); - PERFORM set_config('<%= schema %>.is_in_event_trigger', '', true); - - EXECUTE format('<%= publication_sql %>;', '<%= schema %>', ('shadow__' || _schema || '__' || _table)::name); - ELSE - RAISE EXCEPTION 'table %.% is already electrified', _schema, _table; - END IF; -END; -$function$ LANGUAGE PLPGSQL; - -------------------------------------------------- - -CREATE OR REPLACE FUNCTION <%= schema %>.__table_is_electrified(classid oid, objid oid) -RETURNS boolean AS $function$ -BEGIN - RETURN EXISTS (SELECT id FROM <%= electrified_tracking_table %> WHERE oid = objid); -END; -$function$ LANGUAGE PLPGSQL; - -------------------------------------------------- - -CREATE OR REPLACE FUNCTION <%= schema %>.__create_index_is_electrified(classid oid, objid oid) -RETURNS int8 AS $function$ -DECLARE - _eid int8; -BEGIN - SELECT e.id INTO _eid FROM <%= electrified_tracking_table %> e - INNER JOIN pg_index pi ON e.oid = pi.indrelid - WHERE pi.indexrelid = objid; - RETURN _eid; -END; -$function$ LANGUAGE PLPGSQL; - -------------------------------------------------- - -CREATE OR REPLACE FUNCTION <%= schema %>.ddlx_command_end_handler() -RETURNS EVENT_TRIGGER AS $function$ -DECLARE - _trid int8; - _cmd record; - _capture bool := false; - _table_id int8; -BEGIN - -- Usually, this would be a great place for `pg_trigger_depth()`, but for event triggers that's always 0. - IF (current_setting('electric.is_in_event_trigger', true) = 'true') THEN RETURN; END IF; - RAISE DEBUG 'command_end_handler:: start'; - - FOR _cmd IN SELECT * FROM pg_event_trigger_ddl_commands() - LOOP - -- don't capture create table events, those are inserted by the electrify call - CASE - WHEN _cmd.object_type = 'table' THEN - _capture := _capture OR <%= schema %>.__table_is_electrified(_cmd.classid, _cmd.objid); - WHEN _cmd.object_type = 'table column' THEN - IF <%= schema %>.__table_is_electrified(_cmd.classid, _cmd.objid) THEN - RAISE EXCEPTION 'modifying column of electrified table %', _cmd.object_identity; - END IF; - WHEN _cmd.command_tag = 'CREATE INDEX' THEN - _table_id := <%= schema %>.__create_index_is_electrified(_cmd.classid, _cmd.objid); - IF _table_id THEN - -- capture the index id into a special electrified_index table - -- so that we can check if it's electrified when it's being dropped - -- without this, by the time we get the cmd in the event trigger the - -- index has already been dropped and the lookups required no longer - -- exist - INSERT INTO <%= electrified_index_table %> (id, table_id) VALUES (_cmd.objid, _table_id); - _capture := true; - END IF; - ELSE NULL; - END CASE; - END LOOP; - - IF _capture THEN - _trid := (SELECT <%= schema %>.capture_ddl()); - RAISE DEBUG 'create_active_migration = %', _trid; - - -- We're going to alter multiple tables here. We don't want to re-trigger this function. - PERFORM set_config('<%= schema %>.is_in_event_trigger', 'true', true); - FOR _cmd IN (SELECT * FROM pg_event_trigger_ddl_commands()) - LOOP - RAISE DEBUG ' Current statement touches a % in schema % with objid %', _cmd.object_type, _cmd.schema_name, _cmd.object_identity; - IF _cmd.object_type = 'table' AND <%= schema %>.__table_is_electrified(_cmd.classid, _cmd.objid) THEN - -- Since we're performing these calls only against already-electrified tables, this can only be an "ALTER TABLE" statement - -- which means we won't be creating new tables, which means we don't need to add them to publications - PERFORM <%= schema %>.ddlx_make_or_update_shadow_tables(_cmd.command_tag, _cmd.schema_name, _cmd.objid); - END IF; - END LOOP; - PERFORM set_config('<%= schema %>.is_in_event_trigger', '', true); - END IF; - - RAISE DEBUG 'command_end_handler:: end'; -END; -$function$ LANGUAGE PLPGSQL; - - -CREATE OR REPLACE FUNCTION <%= schema %>.ddlx_sql_drop_handler() -RETURNS EVENT_TRIGGER AS $function$ -DECLARE - _capture bool := false; - _cmd record; - _trid int8; -BEGIN - FOR _cmd IN SELECT * FROM pg_event_trigger_dropped_objects() - LOOP - RAISE DEBUG 'DROP // classid: %; objid: %; objsubid: %, object_type: %', - _cmd.classid, _cmd.objid, _cmd.objsubid, _cmd.object_type; - CASE - WHEN _cmd.object_type = 'table' THEN - IF <%= schema %>.__table_is_electrified(_cmd.classid, _cmd.objid) THEN - RAISE EXCEPTION 'dropping electrified table %', _cmd.object_identity; - END IF; - WHEN _cmd.object_type = 'table column' THEN - IF <%= schema %>.__table_is_electrified(_cmd.classid, _cmd.objid) THEN - RAISE EXCEPTION 'dropping column electrified table %', _cmd.object_identity; - END IF; - WHEN _cmd.object_type = 'index' THEN - IF EXISTS (SELECT id FROM <%= electrified_index_table %> WHERE id = _cmd.objid) THEN - -- clean up the electrified index table - DELETE FROM <%= electrified_index_table %> WHERE id = _cmd.objid; - RAISE WARNING 'index is being dropped %', _capture; - _capture := true; - END IF; - ELSE NULL; - END CASE; - END LOOP; - - IF _capture THEN - _trid := (SELECT <%= schema %>.capture_ddl()); - END IF; -END; -$function$ LANGUAGE PLPGSQL; diff --git a/components/electric/lib/electric/postgres/extension/migrations/20230829000000_acknowledged_client_lsns_table.ex b/components/electric/lib/electric/postgres/extension/migrations/20230829000000_acknowledged_client_lsns_table.ex index 58b91ca0d2..d56eb194de 100644 --- a/components/electric/lib/electric/postgres/extension/migrations/20230829000000_acknowledged_client_lsns_table.ex +++ b/components/electric/lib/electric/postgres/extension/migrations/20230829000000_acknowledged_client_lsns_table.ex @@ -3,26 +3,24 @@ defmodule Electric.Postgres.Extension.Migrations.Migration_20230829000000_Acknow @behaviour Extension.Migration - sql_file = - Path.expand( - "20230829000000_acknowledged_client_lsns_table/deduplicating_trigger.sql", - __DIR__ - ) - - @external_resource sql_file - - @trigger_sql File.read!(sql_file) - @impl true def version, do: 2023_08_29_00_00_00 @impl true - def up(_) do + def up(schema) do table = Extension.acked_client_lsn_table() replicated_table_ddls() ++ [ - @trigger_sql, + Extension.Functions.by_name(:upsert_acknowledged_client_lsn), + """ + CREATE OR REPLACE TRIGGER upsert_acknowledged_client_lsn + BEFORE INSERT ON #{table} + FOR EACH ROW + WHEN (pg_trigger_depth() < 1) + EXECUTE FUNCTION #{schema}.upsert_acknowledged_client_lsn() + """, + "ALTER TABLE #{table} ENABLE REPLICA TRIGGER upsert_acknowledged_client_lsn", Extension.add_table_to_publication_sql(table) ] end diff --git a/components/electric/test/support/extension_case.ex b/components/electric/test/support/extension_case.ex index 8a6de3c49c..0a4194d1d3 100644 --- a/components/electric/test/support/extension_case.ex +++ b/components/electric/test/support/extension_case.ex @@ -50,6 +50,7 @@ defmodule Electric.Extension.Case.Helpers do |> Enum.map(&apply(&1, :version, [])) ExUnit.Assertions.assert({:ok, ^expected_versions} = Extension.migrate(conn)) + ExUnit.Assertions.assert(:ok = Extension.define_functions(conn)) {:ok, oid} = Electric.Replication.Postgres.Client.query_oids(conn) Electric.Postgres.OidDatabase.save_oids(oid) From 99b1424c212169253b2b3333535c0783f2ce5009 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Tue, 3 Oct 2023 13:10:55 +0300 Subject: [PATCH 2/9] Clean up templated SQL routines --- .../electric/postgres/extension/functions.ex | 22 ++---- .../extension/functions/__pg_version.sql.eex | 2 +- .../functions/__primary_key_list.sql.eex | 2 +- .../__resolve_table_from_names.sql.eex | 6 +- .../__session_replication_role.sql.eex | 2 +- .../functions/__table_schema.sql.eex | 2 +- .../functions/alter_shadow_table.sql.eex | 14 ++-- .../functions/assign_default_version.sql.eex | 6 +- .../assign_migration_version.sql.eex | 4 +- .../extension/functions/capture_ddl.sql.eex | 6 +- .../functions/current_transaction_id.sql.eex | 10 +-- .../functions/current_xact_id.sql.eex | 3 +- .../functions/current_xact_ts.sql.eex | 5 +- .../functions/{ => ddlx}/assign.sql.eex | 69 +++++++++---------- .../extension/functions/ddlx/disable.sql.eex | 2 +- .../extension/functions/ddlx/enable.sql.eex | 4 +- .../extension/functions/ddlx/grant.sql.eex | 4 +- .../functions/{ => ddlx}/unassign.sql.eex | 18 ++--- .../extension/functions/electrify.sql.eex | 29 ++++---- .../__validate_table_column_defaults.sql.eex} | 2 +- .../__validate_table_column_types.sql.eex} | 2 +- .../generate_electrified_sql.sql.eex | 0 .../functions/find_fk_to_table.sql.eex | 4 +- .../install_functions_and_triggers.sql.eex | 44 ++++++------ .../functions/migration_version.sql.eex | 4 +- ...rm_reordered_op_installer_function.sql.eex | 42 +++++------ .../string_utils.sql.eex} | 11 ++- .../functions/tx_has_assigned_version.sql.eex | 10 +-- .../upsert_acknowledged_client_lsn.sql.eex | 4 +- 29 files changed, 156 insertions(+), 177 deletions(-) rename components/electric/lib/electric/postgres/extension/functions/{ => ddlx}/assign.sql.eex (79%) rename components/electric/lib/electric/postgres/extension/functions/{ => ddlx}/unassign.sql.eex (77%) rename components/electric/lib/electric/postgres/extension/functions/{validate_table_column_defaults.sql.eex => electrify/__validate_table_column_defaults.sql.eex} (91%) rename components/electric/lib/electric/postgres/extension/functions/{validate_table_column_types.sql.eex => electrify/__validate_table_column_types.sql.eex} (94%) rename components/electric/lib/electric/postgres/extension/functions/{ => electrify}/generate_electrified_sql.sql.eex (100%) rename components/electric/lib/electric/postgres/extension/{migrations/20230512000000_conflict_resolution_triggers/utility_functions.sql => functions/string_utils.sql.eex} (66%) diff --git a/components/electric/lib/electric/postgres/extension/functions.ex b/components/electric/lib/electric/postgres/extension/functions.ex index 4f996c93ba..a667e208bf 100644 --- a/components/electric/lib/electric/postgres/extension/functions.ex +++ b/components/electric/lib/electric/postgres/extension/functions.ex @@ -3,11 +3,12 @@ defmodule Electric.Postgres.Extension.Functions do This module organizes SQL functions that are to be defined in Electric's internal database schema. """ - alias Electric.Postgres.Extension + import Electric.Postgres.Extension + require EEx sql_files = - "functions/*.sql.eex" + "functions/**/*.sql.eex" |> Path.expand(__DIR__) |> Path.wildcard() @@ -16,7 +17,7 @@ defmodule Electric.Postgres.Extension.Functions do @external_resource path name = path |> Path.basename(".sql.eex") |> String.to_atom() - _ = EEx.function_from_file(:def, name, path, [:assigns]) + _ = EEx.function_from_file(:def, name, path, []) name end @@ -56,19 +57,8 @@ defmodule Electric.Postgres.Extension.Functions do """ @spec by_name(name) :: sql def by_name(name) when name in @function_names do - apply(__MODULE__, name, [assigns()]) + apply(__MODULE__, name, []) end - # This map of assigns is the same for all function templates. - defp assigns do - %{ - schema: Extension.schema(), - ddl_table: Extension.ddl_table(), - txid_type: Extension.txid_type(), - txts_type: Extension.txts_type(), - version_table: Extension.version_table(), - electrified_tracking_table: Extension.electrified_tracking_table(), - publication_name: Extension.publication_name() - } - end + defp publication_sql, do: add_table_to_publication_sql("%I.%I") end diff --git a/components/electric/lib/electric/postgres/extension/functions/__pg_version.sql.eex b/components/electric/lib/electric/postgres/extension/functions/__pg_version.sql.eex index dc6e0b88ce..0103127b9a 100644 --- a/components/electric/lib/electric/postgres/extension/functions/__pg_version.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/__pg_version.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.__pg_version() RETURNS integer SECURITY DEFINER AS $function$ +CREATE OR REPLACE FUNCTION <%= schema() %>.__pg_version() RETURNS integer SECURITY DEFINER AS $function$ SELECT setting::int FROM pg_settings WHERE name = 'server_version_num' $function$ LANGUAGE SQL; diff --git a/components/electric/lib/electric/postgres/extension/functions/__primary_key_list.sql.eex b/components/electric/lib/electric/postgres/extension/functions/__primary_key_list.sql.eex index 01e82a1865..e14d58f839 100644 --- a/components/electric/lib/electric/postgres/extension/functions/__primary_key_list.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/__primary_key_list.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.__primary_key_list(target_oid oid) +CREATE OR REPLACE FUNCTION <%= schema() %>.__primary_key_list(target_oid oid) RETURNS text[] SECURITY DEFINER LANGUAGE SQL AS diff --git a/components/electric/lib/electric/postgres/extension/functions/__resolve_table_from_names.sql.eex b/components/electric/lib/electric/postgres/extension/functions/__resolve_table_from_names.sql.eex index 4a5d85bcae..6f020290e5 100644 --- a/components/electric/lib/electric/postgres/extension/functions/__resolve_table_from_names.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/__resolve_table_from_names.sql.eex @@ -1,5 +1,5 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.__resolve_table_from_names( +CREATE OR REPLACE FUNCTION <%= schema() %>.__resolve_table_from_names( name1 text, name2 text, OUT schema_name name, @@ -18,7 +18,7 @@ BEGIN IF array_length(_ident, 1) = 1 THEN table_name := _ident[1]; table_oid := (SELECT quote_ident(table_name)::regclass); - schema_name := <%= @schema %>.__table_schema(table_oid); + schema_name := <%= schema() %>.__table_schema(table_oid); ELSIF array_length(_ident, 1) = 2 THEN schema_name := _ident[1]; table_name := _ident[2]; @@ -29,7 +29,7 @@ BEGIN ELSE table_name := name1; table_oid := (SELECT quote_ident(table_name)::regclass); - schema_name := <%= @schema %>.__table_schema(table_oid); + schema_name := <%= schema() %>.__table_schema(table_oid); END IF; ELSIF name1 IS NOT NULL AND name2 IS NOT NULL THEN table_name := name2; diff --git a/components/electric/lib/electric/postgres/extension/functions/__session_replication_role.sql.eex b/components/electric/lib/electric/postgres/extension/functions/__session_replication_role.sql.eex index 5181bab64a..fb76065020 100644 --- a/components/electric/lib/electric/postgres/extension/functions/__session_replication_role.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/__session_replication_role.sql.eex @@ -10,7 +10,7 @@ -- -- [1]: https://www.postgresql.org/docs/14/runtime-config-client.html#GUC-SESSION-REPLICATION-ROLE -- [2]: https://www.postgresql.org/docs/14/runtime-config-custom.html -CREATE OR REPLACE FUNCTION <%= @schema %>.__session_replication_role(OUT role text) AS $$ +CREATE OR REPLACE FUNCTION <%= schema() %>.__session_replication_role(OUT role text) AS $$ BEGIN SELECT INTO role current_setting('electric.session_replication_role'); EXCEPTION WHEN undefined_object THEN diff --git a/components/electric/lib/electric/postgres/extension/functions/__table_schema.sql.eex b/components/electric/lib/electric/postgres/extension/functions/__table_schema.sql.eex index 5ef541599f..1e2dc721a2 100644 --- a/components/electric/lib/electric/postgres/extension/functions/__table_schema.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/__table_schema.sql.eex @@ -1,5 +1,5 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.__table_schema(table_oid oid) RETURNS name SECURITY DEFINER AS $function$ +CREATE OR REPLACE FUNCTION <%= schema() %>.__table_schema(table_oid oid) RETURNS name SECURITY DEFINER AS $function$ DECLARE _schema name; BEGIN diff --git a/components/electric/lib/electric/postgres/extension/functions/alter_shadow_table.sql.eex b/components/electric/lib/electric/postgres/extension/functions/alter_shadow_table.sql.eex index 5ed2581f2b..c4c5b62f0c 100644 --- a/components/electric/lib/electric/postgres/extension/functions/alter_shadow_table.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/alter_shadow_table.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE PROCEDURE <%= @schema %>.alter_shadow_table( +CREATE OR REPLACE PROCEDURE <%= schema() %>.alter_shadow_table( schema_name name, table_name name, action text, @@ -23,22 +23,22 @@ BEGIN tombstone_table_name := 'tombstone__' || schema_name || '__' || table_name; reordered_column_definition := format(E' ADD COLUMN %I %s,\n', '__reordered_' || col_name, col_type); - shadow_column_definition := format(E' ADD COLUMN %I <%= @schema %>.tag', '_tag_' || col_name); + shadow_column_definition := format(E' ADD COLUMN %I <%= schema() %>.tag', '_tag_' || col_name); tombstone_column_definition := format(E' ADD COLUMN %I %s\n', col_name, col_type); EXECUTE format( - E'ALTER TABLE <%= @schema %>.%I\n%s;', + E'ALTER TABLE <%= schema() %>.%I\n%s;', shadow_table_name, reordered_column_definition || shadow_column_definition ); EXECUTE format( - E'ALTER TABLE <%= @schema %>.%I\n%s;', + E'ALTER TABLE <%= schema() %>.%I\n%s;', tombstone_table_name, tombstone_column_definition ); - SELECT r.table_oid INTO table_oid FROM <%= @schema %>.__resolve_table_from_names(schema_name, table_name) r; - SELECT <%= @schema %>.__primary_key_list(table_oid) INTO primary_key_list; + SELECT r.table_oid INTO table_oid FROM <%= schema() %>.__resolve_table_from_names(schema_name, table_name) r; + SELECT <%= schema() %>.__primary_key_list(table_oid) INTO primary_key_list; SELECT array_agg(c.col_name) INTO non_pk_column_list FROM electric.lookup_columns(table_oid) c WHERE NOT col_primary; @@ -51,7 +51,7 @@ BEGIN explicitly) but safer since `DROP` + `CREATE` can have some unexpected effects. */ - PERFORM <%= @schema %>.install_conflict_resolution_functions(schema_name, table_name, primary_key_list, non_pk_column_list); + PERFORM <%= schema() %>.install_conflict_resolution_functions(schema_name, table_name, primary_key_list, non_pk_column_list); END; $function$; diff --git a/components/electric/lib/electric/postgres/extension/functions/assign_default_version.sql.eex b/components/electric/lib/electric/postgres/extension/functions/assign_default_version.sql.eex index db0c250001..cb2b14abd3 100644 --- a/components/electric/lib/electric/postgres/extension/functions/assign_default_version.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/assign_default_version.sql.eex @@ -1,15 +1,15 @@ -CREATE OR REPLACE PROCEDURE <%= @schema %>.assign_default_version() SECURITY DEFINER AS +CREATE OR REPLACE PROCEDURE <%= schema() %>.assign_default_version() SECURITY DEFINER AS $function$ DECLARE _version text; BEGIN RAISE DEBUG 'checking for migration version'; - IF NOT (<%= @schema %>.tx_has_assigned_version()) THEN + IF NOT (<%= schema() %>.tx_has_assigned_version()) THEN _version := (select to_char(transaction_timestamp(), 'YYYYMMDDHH24MISS_MS')); RAISE DEBUG 'assign default migration version: %', _version; - CALL <%= @schema %>.assign_migration_version(_version, 0); + CALL <%= schema() %>.assign_migration_version(_version, 0); ELSE RAISE DEBUG 'migration has version %', _version; END IF; diff --git a/components/electric/lib/electric/postgres/extension/functions/assign_migration_version.sql.eex b/components/electric/lib/electric/postgres/extension/functions/assign_migration_version.sql.eex index 5d8f6988bf..c0ccfcd934 100644 --- a/components/electric/lib/electric/postgres/extension/functions/assign_migration_version.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/assign_migration_version.sql.eex @@ -1,9 +1,9 @@ -CREATE OR REPLACE PROCEDURE <%= @schema %>.assign_migration_version(_version text, _priority int) SECURITY DEFINER AS $function$ +CREATE OR REPLACE PROCEDURE <%= schema() %>.assign_migration_version(_version text, _priority int) SECURITY DEFINER AS $function$ BEGIN -- if there's already a version for this tx, then overwrite it only if the priority of -- this version is > the existing one; - INSERT INTO <%= @version_table %> AS v (version, priority) VALUES (_version, _priority::int2) + INSERT INTO <%= version_table() %> AS v (version, priority) VALUES (_version, _priority::int2) ON CONFLICT (txid, txts) DO UPDATE SET version = EXCLUDED.version, priority = EXCLUDED.priority WHERE EXCLUDED.priority > v.priority; diff --git a/components/electric/lib/electric/postgres/extension/functions/capture_ddl.sql.eex b/components/electric/lib/electric/postgres/extension/functions/capture_ddl.sql.eex index 2a92381318..885cc831ad 100644 --- a/components/electric/lib/electric/postgres/extension/functions/capture_ddl.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/capture_ddl.sql.eex @@ -1,16 +1,16 @@ -CREATE OR REPLACE PROCEDURE <%= @schema %>.capture_ddl(_query text DEFAULT NULL) SECURITY DEFINER AS +CREATE OR REPLACE PROCEDURE <%= schema() %>.capture_ddl(_query text DEFAULT NULL) SECURITY DEFINER AS $function$ BEGIN IF _query IS NULL THEN _query := current_query(); END IF; - CALL <%= @schema %>.assign_default_version(); + CALL <%= schema() %>.assign_default_version(); RAISE DEBUG 'capture migration: %', _query; - INSERT INTO <%= @ddl_table %> (query) + INSERT INTO <%= ddl_table() %> (query) VALUES (_query) ON CONFLICT ON CONSTRAINT ddl_table_unique_migrations DO NOTHING; diff --git a/components/electric/lib/electric/postgres/extension/functions/current_transaction_id.sql.eex b/components/electric/lib/electric/postgres/extension/functions/current_transaction_id.sql.eex index f412539954..a49535fbe8 100644 --- a/components/electric/lib/electric/postgres/extension/functions/current_transaction_id.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/current_transaction_id.sql.eex @@ -1,10 +1,10 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.current_transaction_id( - OUT txid <%= @txid_type %>, - OUT txts <%= @txts_type %> +CREATE OR REPLACE FUNCTION <%= schema() %>.current_transaction_id( + OUT txid <%= txid_type() %>, + OUT txts <%= txts_type() %> ) SECURITY DEFINER AS $function$ BEGIN - txid := (SELECT <%= @schema %>.current_xact_id()); - txts := (SELECT <%= @schema %>.current_xact_ts()); + txid := (SELECT <%= schema() %>.current_xact_id()); + txts := (SELECT <%= schema() %>.current_xact_ts()); END; $function$ LANGUAGE PLPGSQL; diff --git a/components/electric/lib/electric/postgres/extension/functions/current_xact_id.sql.eex b/components/electric/lib/electric/postgres/extension/functions/current_xact_id.sql.eex index 376e0391f5..d61098ff22 100644 --- a/components/electric/lib/electric/postgres/extension/functions/current_xact_id.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/current_xact_id.sql.eex @@ -1,4 +1,5 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.current_xact_id() RETURNS <%= @txid_type %> SECURITY DEFINER AS $function$ +CREATE OR REPLACE FUNCTION <%= schema() %>.current_xact_id() +RETURNS <%= txid_type() %> SECURITY DEFINER AS $function$ BEGIN RETURN pg_current_xact_id(); END; diff --git a/components/electric/lib/electric/postgres/extension/functions/current_xact_ts.sql.eex b/components/electric/lib/electric/postgres/extension/functions/current_xact_ts.sql.eex index 7abc5919bd..23b5aa84b1 100644 --- a/components/electric/lib/electric/postgres/extension/functions/current_xact_ts.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/current_xact_ts.sql.eex @@ -1,5 +1,6 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.current_xact_ts() RETURNS <%= @txts_type %> SECURITY DEFINER AS $function$ +CREATE OR REPLACE FUNCTION <%= schema() %>.current_xact_ts() +RETURNS <%= txts_type() %> SECURITY DEFINER AS $function$ BEGIN - RETURN (extract(epoch from transaction_timestamp()) * 1000000)::<%= @txts_type %>; + RETURN (extract(epoch from transaction_timestamp()) * 1000000)::<%= txts_type() %>; END; $function$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE; diff --git a/components/electric/lib/electric/postgres/extension/functions/assign.sql.eex b/components/electric/lib/electric/postgres/extension/functions/ddlx/assign.sql.eex similarity index 79% rename from components/electric/lib/electric/postgres/extension/functions/assign.sql.eex rename to components/electric/lib/electric/postgres/extension/functions/ddlx/assign.sql.eex index f4a30d2a49..2470fca85d 100644 --- a/components/electric/lib/electric/postgres/extension/functions/assign.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/ddlx/assign.sql.eex @@ -1,9 +1,4 @@ --- vim:ft=sql - -<% roles_table = Electric.Postgres.Extension.roles_table() %> -<% assignments_table = Electric.Postgres.Extension.assignments_table() %> - -CREATE OR REPLACE PROCEDURE <%= @schema %>.assign( +CREATE OR REPLACE PROCEDURE <%= schema() %>.assign( assign_table_full_name text, scope text, user_column_name text, @@ -63,12 +58,12 @@ BEGIN SELECT n[1], n[2] INTO assign_schema, assign_table FROM parse_ident(assign_table_full_name) n; -- gets the columns and types for the assign_table's primary key - INSERT INTO pkeys SELECT * from <%= @schema %>.find_pk(assign_schema, assign_table); + INSERT INTO pkeys SELECT * from <%= schema() %>.find_pk(assign_schema, assign_table); SELECT * FROM pkeys LIMIT 1 INTO primary_key; -- gets the foreign key pointing to the user - INSERT INTO user_fkeys SELECT * from <%= @schema %>.find_fk_for_column(assign_schema,assign_table, user_column_name); + INSERT INTO user_fkeys SELECT * from <%= schema() %>.find_fk_for_column(assign_schema,assign_table, user_column_name); SELECT COUNT(*) FROM user_fkeys INTO user_key_count; IF user_key_count = 0 THEN @@ -116,7 +111,7 @@ BEGIN -- reads the foreign key for the scope if it exists IF NOT scope IS NULL THEN - INSERT INTO scope_fkeys SELECT * from <%= @schema %>.find_fk_to_table(assign_schema,assign_table, scope); + INSERT INTO scope_fkeys SELECT * from <%= schema() %>.find_fk_to_table(assign_schema,assign_table, scope); SELECT COUNT(*) FROM scope_fkeys INTO scope_key_count; IF scope_key_count > 1 THEN @@ -140,7 +135,7 @@ BEGIN END IF; -- Creates the assignment itself. - INSERT INTO <%= assignments_table %> (table_name, scope_table, user_column, role_name, role_column, if_fn) + INSERT INTO <%= assignments_table() %> (table_name, scope_table, user_column, role_name, role_column, if_fn) VALUES (assign_table_full_name, scope_table_not_null, user_column_name, role_name_not_null, role_column_not_null, if_fn) RETURNING id INTO assignment_id; @@ -173,18 +168,18 @@ BEGIN And it has a foreign key pointing to the role in electric.roles which it will delete with a trigger. */ - EXECUTE format('CREATE TABLE IF NOT EXISTS <%= @schema %>.assignment_%s_join ( + EXECUTE format('CREATE TABLE IF NOT EXISTS <%= schema() %>.assignment_%s_join ( id uuid PRIMARY KEY DEFAULT gen_random_uuid(), user_id %s, assignment_id uuid, role_id uuid, FOREIGN KEY(role_id) - REFERENCES <%= roles_table %> (id), + REFERENCES <%= roles_table() %> (id), FOREIGN KEY(user_id) REFERENCES %s.%s(%s) ON DELETE CASCADE, FOREIGN KEY(assignment_id) - REFERENCES <%= assignments_table %> (id) + REFERENCES <%= assignments_table() %> (id) ON DELETE CASCADE );', assignment_name, @@ -197,7 +192,7 @@ BEGIN -- Adds a foreign key to the join table pointing to the assign_table for counter in 1..ARRAY_LENGTH(primary_key.columns, 1) loop - EXECUTE format('ALTER TABLE <%= @schema %>.assignment_%s_join ADD COLUMN IF NOT EXISTS %s_%s %s;', + EXECUTE format('ALTER TABLE <%= schema() %>.assignment_%s_join ADD COLUMN IF NOT EXISTS %s_%s %s;', assignment_name, assign_table, primary_key.columns[counter], @@ -205,7 +200,7 @@ BEGIN ); end loop; - EXECUTE format('ALTER TABLE <%= @schema %>.assignment_%s_join + EXECUTE format('ALTER TABLE <%= schema() %>.assignment_%s_join ADD CONSTRAINT electric_%s_join_%s_fk FOREIGN KEY (%s_%s) REFERENCES %s.%s(%s) @@ -224,29 +219,29 @@ BEGIN -- when there is no scope IF scope IS NULL THEN - EXECUTE format(E'CREATE OR REPLACE FUNCTION <%= @schema %>.upsert_role_%1$s() RETURNS TRIGGER SECURITY DEFINER + EXECUTE format(E'CREATE OR REPLACE FUNCTION <%= schema() %>.upsert_role_%1$s() RETURNS TRIGGER SECURITY DEFINER AS $%2$s$ DECLARE role_key uuid; join_key uuid; BEGIN - SELECT id, role_id FROM <%= @schema %>.assignment_%1$s_join WHERE assignment_id = \'%4$s\' AND ( %5$s_%6$s ) = ( NEW.%7$s ) INTO join_key, role_key; + SELECT id, role_id FROM <%= schema() %>.assignment_%1$s_join WHERE assignment_id = \'%4$s\' AND ( %5$s_%6$s ) = ( NEW.%7$s ) INTO join_key, role_key; IF ( %8$s ) THEN IF join_key IS NULL THEN - INSERT INTO <%= roles_table %> (user_id, role) + INSERT INTO <%= roles_table() %> (user_id, role) VALUES (NEW.%9$s, %10$s) returning id INTO role_key; - INSERT INTO <%= @schema %>.assignment_%1$s_join (user_id, %5$s_%6$s, role_id, assignment_id) + INSERT INTO <%= schema() %>.assignment_%1$s_join (user_id, %5$s_%6$s, role_id, assignment_id) VALUES (NEW.%9$s, NEW.%7$s, role_key, \'%4$s\'); ELSE - UPDATE <%= @schema %>.assignment_%1$s_join SET user_id = NEW.%9$s + UPDATE <%= schema() %>.assignment_%1$s_join SET user_id = NEW.%9$s WHERE id = join_key; - UPDATE <%= roles_table %> SET (user_id, role) = (NEW.%9$s, %10s) + UPDATE <%= roles_table() %> SET (user_id, role) = (NEW.%9$s, %10s) WHERE id = role_key; END IF; ELSE IF NOT join_key IS NULL THEN - DELETE FROM <%= @schema %>.assignment_%1$s_join WHERE id = join_key; + DELETE FROM <%= schema() %>.assignment_%1$s_join WHERE id = join_key; END IF; END IF; RETURN NEW; @@ -280,14 +275,14 @@ BEGIN ELSE for counter in 1..ARRAY_LENGTH(scope_key.from_columns, 1) loop - EXECUTE format('ALTER TABLE <%= @schema %>.assignment_%s_join ADD COLUMN IF NOT EXISTS %s %s;', + EXECUTE format('ALTER TABLE <%= schema() %>.assignment_%s_join ADD COLUMN IF NOT EXISTS %s %s;', assignment_name, scope_key.from_columns[counter], scope_key.to_types[counter] ); end loop; - EXECUTE format('ALTER TABLE <%= @schema %>.assignment_%s_join + EXECUTE format('ALTER TABLE <%= schema() %>.assignment_%s_join ADD CONSTRAINT electric_%s_join_scope_fk FOREIGN KEY (%s) REFERENCES %s.%s(%s) @@ -300,7 +295,7 @@ BEGIN ARRAY_TO_STRING(scope_key.to_columns, ', ') ); - EXECUTE format(E'CREATE OR REPLACE FUNCTION <%= @schema %>.upsert_role_%1$s() RETURNS TRIGGER SECURITY DEFINER + EXECUTE format(E'CREATE OR REPLACE FUNCTION <%= schema() %>.upsert_role_%1$s() RETURNS TRIGGER SECURITY DEFINER AS $%2$s$ DECLARE scope_key TEXT; @@ -312,22 +307,22 @@ BEGIN scope_list := ARRAY[NEW.%3$s::text]; scope_key := ARRAY_TO_STRING(scope_list, \', \' ); - SELECT id, role_id FROM <%= @schema %>.assignment_%1$s_join WHERE assignment_id = \'%4$s\' AND ( %5$s_%6$s ) = ( NEW.%7$s ) INTO join_key, role_key; + SELECT id, role_id FROM <%= schema() %>.assignment_%1$s_join WHERE assignment_id = \'%4$s\' AND ( %5$s_%6$s ) = ( NEW.%7$s ) INTO join_key, role_key; IF ( %8$s ) THEN IF join_key IS NULL THEN - INSERT INTO <%= roles_table %> (user_id, role, scope_table, scope_id) + INSERT INTO <%= roles_table() %> (user_id, role, scope_table, scope_id) VALUES (NEW.%9$s, %10$s, \'%11$s\', scope_key) returning id INTO role_key; - INSERT INTO <%= @schema %>.assignment_%1$s_join (user_id, %12$s, %5$s_%6$s, role_id, assignment_id) + INSERT INTO <%= schema() %>.assignment_%1$s_join (user_id, %12$s, %5$s_%6$s, role_id, assignment_id) VALUES (NEW.%9$s, NEW.%13$s, NEW.%7$s, role_key, \'%4$s\'); ELSE - UPDATE <%= @schema %>.assignment_%1$s_join SET (user_id, %12$s) + UPDATE <%= schema() %>.assignment_%1$s_join SET (user_id, %12$s) = (NEW.%9$s, NEW.%13$s) WHERE id = join_key; - UPDATE <%= roles_table %> SET (user_id, role, scope_table, scope_id) + UPDATE <%= roles_table() %> SET (user_id, role, scope_table, scope_id) = (NEW.%9$s, %10$s, \'%11$s\', scope_key) WHERE id = role_key; END IF; ELSE IF NOT join_key IS NULL THEN - DELETE FROM <%= @schema %>.assignment_%1$s_join WHERE id = join_key; + DELETE FROM <%= schema() %>.assignment_%1$s_join WHERE id = join_key; END IF; END IF; RETURN NEW; @@ -363,10 +358,10 @@ BEGIN END IF; -- adds a trigger to the join table that deletes the role itself - EXECUTE format(E'CREATE OR REPLACE FUNCTION <%= @schema %>.cleanup_role_%s() RETURNS TRIGGER SECURITY DEFINER + EXECUTE format(E'CREATE OR REPLACE FUNCTION <%= schema() %>.cleanup_role_%s() RETURNS TRIGGER SECURITY DEFINER AS $%s$ BEGIN - DELETE FROM <%= roles_table %> WHERE id = OLD.role_id; + DELETE FROM <%= roles_table() %> WHERE id = OLD.role_id; RETURN OLD; END; $%s$ LANGUAGE plpgsql;', @@ -376,9 +371,9 @@ BEGIN ); EXECUTE format('CREATE OR REPLACE TRIGGER electric_cleanup_role_%s - AFTER DELETE ON <%= @schema %>.assignment_%s_join + AFTER DELETE ON <%= schema() %>.assignment_%s_join FOR EACH ROW - EXECUTE FUNCTION <%= @schema %>.cleanup_role_%s();', + EXECUTE FUNCTION <%= schema() %>.cleanup_role_%s();', assignment_name, assignment_name, assignment_name @@ -388,7 +383,7 @@ BEGIN EXECUTE format('CREATE OR REPLACE TRIGGER electric_insert_role_%s AFTER INSERT ON %s FOR EACH ROW - EXECUTE FUNCTION <%= @schema %>.upsert_role_%s();', + EXECUTE FUNCTION <%= schema() %>.upsert_role_%s();', assignment_name, assign_table, assignment_name @@ -397,7 +392,7 @@ BEGIN EXECUTE format('CREATE OR REPLACE TRIGGER electric_update_role_%s AFTER UPDATE ON %s FOR EACH ROW - EXECUTE FUNCTION <%= @schema %>.upsert_role_%s();', + EXECUTE FUNCTION <%= schema() %>.upsert_role_%s();', assignment_name, assign_table, assignment_name diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/disable.sql.eex b/components/electric/lib/electric/postgres/extension/functions/ddlx/disable.sql.eex index 4251195baf..6c281334d9 100644 --- a/components/electric/lib/electric/postgres/extension/functions/ddlx/disable.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/ddlx/disable.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE PROCEDURE <%= schema %>.disable(table_name text) SECURITY DEFINER AS +CREATE OR REPLACE PROCEDURE <%= schema() %>.disable(table_name text) SECURITY DEFINER AS $$ BEGIN END; diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/enable.sql.eex b/components/electric/lib/electric/postgres/extension/functions/ddlx/enable.sql.eex index 59103c4043..622b3636b4 100644 --- a/components/electric/lib/electric/postgres/extension/functions/ddlx/enable.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/ddlx/enable.sql.eex @@ -1,7 +1,7 @@ -CREATE OR REPLACE PROCEDURE <%= schema %>.enable(table_name text) SECURITY DEFINER AS +CREATE OR REPLACE PROCEDURE <%= schema() %>.enable(table_name text) SECURITY DEFINER AS $$ BEGIN - CALL <%= schema %>.electrify(table_name); + CALL <%= schema() %>.electrify(table_name); END; $$ LANGUAGE plpgsql; diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/grant.sql.eex b/components/electric/lib/electric/postgres/extension/functions/ddlx/grant.sql.eex index 5613049add..0a539e67c3 100644 --- a/components/electric/lib/electric/postgres/extension/functions/ddlx/grant.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/ddlx/grant.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE PROCEDURE <%= schema %>.grant( +CREATE OR REPLACE PROCEDURE <%= schema() %>.grant( privilege_name text, on_table_name text, role_name text, @@ -14,7 +14,7 @@ CREATE OR REPLACE PROCEDURE <%= schema %>.grant( BEGIN FOREACH col IN ARRAY columns LOOP - INSERT INTO <%= Extension.grants_table() %> ( privilege, on_table, role , column_name, scope, using_path, check_fn) + INSERT INTO <%= grants_table() %> ( privilege, on_table, role , column_name, scope, using_path, check_fn) VALUES (privilege_name, on_table_name, role_name, col, scope_name, using_path, check_fn) ON CONFLICT ON CONSTRAINT grants_pkey DO UPDATE SET (using_path, check_fn) = (EXCLUDED.using_path, EXCLUDED.check_fn); diff --git a/components/electric/lib/electric/postgres/extension/functions/unassign.sql.eex b/components/electric/lib/electric/postgres/extension/functions/ddlx/unassign.sql.eex similarity index 77% rename from components/electric/lib/electric/postgres/extension/functions/unassign.sql.eex rename to components/electric/lib/electric/postgres/extension/functions/ddlx/unassign.sql.eex index 0672ff4d41..efc66eeb5c 100644 --- a/components/electric/lib/electric/postgres/extension/functions/unassign.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/ddlx/unassign.sql.eex @@ -1,8 +1,4 @@ --- vim:ft=sql - -<% assignments_table = Electric.Postgres.Extension.assignments_table() %> - -CREATE OR REPLACE PROCEDURE <%= @schema %>.unassign( +CREATE OR REPLACE PROCEDURE <%= schema() %>.unassign( assign_table_full_name text, scope text, user_column_name text, @@ -25,7 +21,7 @@ BEGIN -- SELECT n[1], n[2] INTO assign_schema, assign_table FROM parse_ident(assign_table_full_name) n; - SELECT id FROM <%= assignments_table %> + SELECT id FROM <%= assignments_table() %> WHERE table_name = assign_table_full_name AND scope_table = scope_table_not_null AND user_column = user_column_name @@ -36,7 +32,7 @@ BEGIN assignment_name = REPLACE(format('%s', assignment_id), '-', '_'); -- remove triggers - EXECUTE format('DROP TRIGGER IF EXISTS electric_cleanup_role_%s ON <%= @schema %>.assignment_%s_join;', + EXECUTE format('DROP TRIGGER IF EXISTS electric_cleanup_role_%s ON <%= schema() %>.assignment_%s_join;', assignment_name, assignment_name ); @@ -52,21 +48,21 @@ BEGIN ); -- remove functions - EXECUTE format('DROP FUNCTION IF EXISTS <%= @schema %>.cleanup_role_%s;', + EXECUTE format('DROP FUNCTION IF EXISTS <%= schema() %>.cleanup_role_%s;', assignment_name ); - EXECUTE format('DROP FUNCTION IF EXISTS <%= @schema %>.upsert_role_%s;', + EXECUTE format('DROP FUNCTION IF EXISTS <%= schema() %>.upsert_role_%s;', assignment_name ); -- remove join table - EXECUTE format('DROP TABLE IF EXISTS <%= @schema %>.assignment_%s_join;', + EXECUTE format('DROP TABLE IF EXISTS <%= schema() %>.assignment_%s_join;', assignment_name ); -- remove assignment - DELETE FROM <%= assignments_table %> WHERE id = assignment_id; + DELETE FROM <%= assignments_table() %> WHERE id = assignment_id; END; $$ LANGUAGE plpgsql; diff --git a/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex b/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex index 25911857a8..540e481f93 100644 --- a/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex @@ -1,5 +1,4 @@ -<% publication_sql = Extension.add_table_to_publication_sql("%I.%I") %> -CREATE OR REPLACE PROCEDURE <%= @schema %>.electrify( +CREATE OR REPLACE PROCEDURE <%= schema() %>.electrify( name1 text, name2 text DEFAULT NULL ) SECURITY DEFINER AS $function$ @@ -14,7 +13,7 @@ BEGIN table_name, schema_name, table_oid INTO _table, _schema, _oid - FROM <%= @schema %>.__resolve_table_from_names(name1, name2); + FROM <%= schema() %>.__resolve_table_from_names(name1, name2); _quoted_name := format('%I.%I', _schema, _table); _oid := (SELECT _quoted_name::regclass); @@ -24,37 +23,37 @@ BEGIN END IF; IF NOT EXISTS ( - SELECT oid FROM <%= @electrified_tracking_table %> WHERE schema_name = _schema AND table_name = _table LIMIT 1 + SELECT oid FROM <%= electrified_tracking_table() %> WHERE schema_name = _schema AND table_name = _table LIMIT 1 ) THEN RAISE NOTICE 'Electrify table %', _quoted_name; - CALL <%= @schema %>.__validate_table_column_types(_quoted_name); - CALL <%= @schema %>.__validate_table_column_defaults(_quoted_name); + CALL <%= schema() %>.__validate_table_column_types(_quoted_name); + CALL <%= schema() %>.__validate_table_column_defaults(_quoted_name); -- insert the required ddl into the migrations table - SELECT <%= @schema %>.generate_electrified_sql(_oid) INTO _create_sql; + SELECT <%= schema() %>.generate_electrified_sql(_oid) INTO _create_sql; RAISE DEBUG '%', _create_sql; - CALL <%= @schema %>.capture_ddl(_create_sql); + CALL <%= schema() %>.capture_ddl(_create_sql); EXECUTE format('ALTER TABLE %I.%I REPLICA IDENTITY FULL;', _schema, _table); - INSERT INTO <%= @electrified_tracking_table %> + INSERT INTO <%= electrified_tracking_table() %> (schema_name, table_name, oid) VALUES (_schema, _table, _oid); - EXECUTE format('<%= publication_sql %>;', _schema, _table); + EXECUTE format('<%= publication_sql() %>;', _schema, _table); -- We want to disable any possible hooks from `CREATE TABLE` statements - PERFORM set_config('<%= @schema %>.is_in_event_trigger', 'true', true); - PERFORM <%= @schema %>.ddlx_make_or_update_shadow_tables('CREATE TABLE', _schema, _oid); - PERFORM set_config('<%= @schema %>.is_in_event_trigger', '', true); + PERFORM set_config('<%= schema() %>.is_in_event_trigger', 'true', true); + PERFORM <%= schema() %>.ddlx_make_or_update_shadow_tables('CREATE TABLE', _schema, _oid); + PERFORM set_config('<%= schema() %>.is_in_event_trigger', '', true); EXECUTE format( - '<%= publication_sql %>;', - '<%= @schema %>', + '<%= publication_sql() %>;', + '<%= schema() %>', ('shadow__' || _schema || '__' || _table)::name ); ELSE diff --git a/components/electric/lib/electric/postgres/extension/functions/validate_table_column_defaults.sql.eex b/components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_defaults.sql.eex similarity index 91% rename from components/electric/lib/electric/postgres/extension/functions/validate_table_column_defaults.sql.eex rename to components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_defaults.sql.eex index 615aea6a18..f7dbfbf4b1 100644 --- a/components/electric/lib/electric/postgres/extension/functions/validate_table_column_defaults.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_defaults.sql.eex @@ -1,7 +1,7 @@ -- This function validates each column of the table that's being electrified -- and aborts electrification if any column has DEFAULT expression. -CREATE OR REPLACE PROCEDURE <%= @schema %>.__validate_table_column_defaults(table_name text) +CREATE OR REPLACE PROCEDURE <%= schema() %>.__validate_table_column_defaults(table_name text) SECURITY DEFINER AS $function$ DECLARE _col_name text; diff --git a/components/electric/lib/electric/postgres/extension/functions/validate_table_column_types.sql.eex b/components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_types.sql.eex similarity index 94% rename from components/electric/lib/electric/postgres/extension/functions/validate_table_column_types.sql.eex rename to components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_types.sql.eex index e9a81cafef..a0502f6a38 100644 --- a/components/electric/lib/electric/postgres/extension/functions/validate_table_column_types.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_types.sql.eex @@ -8,7 +8,7 @@ |> Enum.join(",") %> -CREATE OR REPLACE PROCEDURE <%= @schema %>.__validate_table_column_types(table_name text) +CREATE OR REPLACE PROCEDURE <%= schema() %>.__validate_table_column_types(table_name text) SECURITY DEFINER AS $function$ DECLARE _col_name text; diff --git a/components/electric/lib/electric/postgres/extension/functions/generate_electrified_sql.sql.eex b/components/electric/lib/electric/postgres/extension/functions/electrify/generate_electrified_sql.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/generate_electrified_sql.sql.eex rename to components/electric/lib/electric/postgres/extension/functions/electrify/generate_electrified_sql.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/find_fk_to_table.sql.eex b/components/electric/lib/electric/postgres/extension/functions/find_fk_to_table.sql.eex index 4a2b74c307..577120e134 100644 --- a/components/electric/lib/electric/postgres/extension/functions/find_fk_to_table.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/find_fk_to_table.sql.eex @@ -1,6 +1,4 @@ --- vim:ft=sql - -CREATE OR REPLACE FUNCTION <%= @schema %>.find_fk_to_table( +CREATE OR REPLACE FUNCTION <%= schema() %>.find_fk_to_table( src_schema text, src_table text, dst_table text diff --git a/components/electric/lib/electric/postgres/extension/functions/install_functions_and_triggers.sql.eex b/components/electric/lib/electric/postgres/extension/functions/install_functions_and_triggers.sql.eex index 42fc3c7490..edeeb1dc23 100644 --- a/components/electric/lib/electric/postgres/extension/functions/install_functions_and_triggers.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/install_functions_and_triggers.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.install_functions_and_triggers(schema_name TEXT, table_name TEXT, primary_key_list TEXT[], non_pk_column_list TEXT[]) +CREATE OR REPLACE FUNCTION <%= schema() %>.install_functions_and_triggers(schema_name TEXT, table_name TEXT, primary_key_list TEXT[], non_pk_column_list TEXT[]) RETURNS VOID LANGUAGE PLPGSQL AS $function$ @@ -8,14 +8,14 @@ DECLARE generated_functions JSONB; BEGIN -- Install function to be used in the triggers - generated_functions := <%= @schema %>.install_conflict_resolution_functions(schema_name, table_name, primary_key_list, non_pk_column_list); + generated_functions := <%= schema() %>.install_conflict_resolution_functions(schema_name, table_name, primary_key_list, non_pk_column_list); -- Install actual triggers EXECUTE format($$ CREATE OR REPLACE TRIGGER as_first__save_deleted_rows_to_tombstone_table AFTER DELETE ON %s FOR EACH ROW - EXECUTE PROCEDURE <%= @schema %>.%I() + EXECUTE PROCEDURE <%= schema() %>.%I() $$, full_table_identifier, generated_functions->>'generate_tombstone_entry'); EXECUTE format($$ ALTER TABLE %s ENABLE ALWAYS TRIGGER as_first__save_deleted_rows_to_tombstone_table $$, full_table_identifier); @@ -23,8 +23,8 @@ BEGIN CREATE OR REPLACE TRIGGER postgres_write__upsert_generate_shadow_rows BEFORE INSERT OR UPDATE ON %s FOR EACH ROW - WHEN (<%= @schema %>.__session_replication_role() <> 'replica') - EXECUTE PROCEDURE <%= @schema %>.%I(); + WHEN (<%= schema() %>.__session_replication_role() <> 'replica') + EXECUTE PROCEDURE <%= schema() %>.%I(); $$, full_table_identifier, generated_functions->>'create_shadow_row_from_upsert'); EXECUTE format($$ ALTER TABLE %s ENABLE TRIGGER postgres_write__upsert_generate_shadow_rows $$, full_table_identifier); @@ -33,50 +33,50 @@ BEGIN CREATE OR REPLACE TRIGGER postgres_write__delete_generate_shadow_rows BEFORE DELETE ON %s FOR EACH ROW - WHEN (<%= @schema %>.__session_replication_role() <> 'replica') - EXECUTE PROCEDURE <%= @schema %>.%I(); + WHEN (<%= schema() %>.__session_replication_role() <> 'replica') + EXECUTE PROCEDURE <%= schema() %>.%I(); $$, full_table_identifier, generated_functions->>'update_shadow_row_from_delete'); EXECUTE format($$ ALTER TABLE %s ENABLE TRIGGER postgres_write__delete_generate_shadow_rows $$, full_table_identifier); - EXECUTE format($$ DROP TRIGGER IF EXISTS postgres_write__write_resolved_tags ON <%= @schema %>.%I $$, shadow_table_name); + EXECUTE format($$ DROP TRIGGER IF EXISTS postgres_write__write_resolved_tags ON <%= schema() %>.%I $$, shadow_table_name); EXECUTE format($$ CREATE CONSTRAINT TRIGGER postgres_write__write_resolved_tags - AFTER UPDATE ON <%= @schema %>.%I + AFTER UPDATE ON <%= schema() %>.%I DEFERRABLE INITIALLY DEFERRED FOR EACH ROW - WHEN (<%= @schema %>.__session_replication_role() <> 'replica' AND NOT NEW._resolved) - EXECUTE PROCEDURE <%= @schema %>.%I(); + WHEN (<%= schema() %>.__session_replication_role() <> 'replica' AND NOT NEW._resolved) + EXECUTE PROCEDURE <%= schema() %>.%I(); $$, shadow_table_name, generated_functions->>'write_correct_max_tag'); - EXECUTE format($$ ALTER TABLE <%= @schema %>.%I ENABLE TRIGGER postgres_write__write_resolved_tags $$, shadow_table_name); + EXECUTE format($$ ALTER TABLE <%= schema() %>.%I ENABLE TRIGGER postgres_write__write_resolved_tags $$, shadow_table_name); EXECUTE format($$ CREATE OR REPLACE TRIGGER satellite_write__upsert_rows - BEFORE INSERT ON <%= @schema %>.%I + BEFORE INSERT ON <%= schema() %>.%I FOR EACH ROW - WHEN (<%= @schema %>.__session_replication_role() = 'replica' AND pg_trigger_depth() < 1 AND NEW._currently_reordering IS NULL) - EXECUTE PROCEDURE <%= @schema %>.%I(); + WHEN (<%= schema() %>.__session_replication_role() = 'replica' AND pg_trigger_depth() < 1 AND NEW._currently_reordering IS NULL) + EXECUTE PROCEDURE <%= schema() %>.%I(); $$, shadow_table_name, generated_functions->>'shadow_insert_to_upsert'); - EXECUTE format($$ ALTER TABLE <%= @schema %>.%I ENABLE ALWAYS TRIGGER satellite_write__upsert_rows $$, shadow_table_name); + EXECUTE format($$ ALTER TABLE <%= schema() %>.%I ENABLE ALWAYS TRIGGER satellite_write__upsert_rows $$, shadow_table_name); EXECUTE format($$ CREATE OR REPLACE TRIGGER satellite_write__resolve_observed_tags - BEFORE UPDATE ON <%= @schema %>.%I + BEFORE UPDATE ON <%= schema() %>.%I FOR EACH ROW - WHEN (<%= @schema %>.__session_replication_role() = 'replica' AND NEW._currently_reordering IS NULL) - EXECUTE PROCEDURE <%= @schema %>.%I(); + WHEN (<%= schema() %>.__session_replication_role() = 'replica' AND NEW._currently_reordering IS NULL) + EXECUTE PROCEDURE <%= schema() %>.%I(); $$, shadow_table_name, generated_functions->>'resolve_observed_tags'); - EXECUTE format($$ ALTER TABLE <%= @schema %>.%I ENABLE ALWAYS TRIGGER satellite_write__resolve_observed_tags $$, shadow_table_name); + EXECUTE format($$ ALTER TABLE <%= schema() %>.%I ENABLE ALWAYS TRIGGER satellite_write__resolve_observed_tags $$, shadow_table_name); EXECUTE format($$ CREATE OR REPLACE TRIGGER satellite_write__save_operation_for_reordering BEFORE INSERT OR UPDATE ON %s FOR EACH ROW - WHEN (<%= @schema %>.__session_replication_role() = 'replica' AND pg_trigger_depth() < 1) - EXECUTE PROCEDURE <%= @schema %>.%I(); + WHEN (<%= schema() %>.__session_replication_role() = 'replica' AND pg_trigger_depth() < 1) + EXECUTE PROCEDURE <%= schema() %>.%I(); $$, full_table_identifier, generated_functions->>'reorder_main_op'); EXECUTE format($$ ALTER TABLE %s ENABLE ALWAYS TRIGGER satellite_write__save_operation_for_reordering $$, full_table_identifier); diff --git a/components/electric/lib/electric/postgres/extension/functions/migration_version.sql.eex b/components/electric/lib/electric/postgres/extension/functions/migration_version.sql.eex index f7c5ac26b3..42ba63217c 100644 --- a/components/electric/lib/electric/postgres/extension/functions/migration_version.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/migration_version.sql.eex @@ -1,7 +1,7 @@ -CREATE OR REPLACE PROCEDURE <%= @schema %>.migration_version(_version text) SECURITY DEFINER AS $function$ +CREATE OR REPLACE PROCEDURE <%= schema() %>.migration_version(_version text) SECURITY DEFINER AS $function$ BEGIN -- this is the user-facing api, so the priority of the version is set to the highest possible value - CALL <%= @schema %>.assign_migration_version(_version, 32767); + CALL <%= schema() %>.assign_migration_version(_version, 32767); END; $function$ LANGUAGE PLPGSQL; diff --git a/components/electric/lib/electric/postgres/extension/functions/perform_reordered_op_installer_function.sql.eex b/components/electric/lib/electric/postgres/extension/functions/perform_reordered_op_installer_function.sql.eex index 801c643eb3..18184d04af 100644 --- a/components/electric/lib/electric/postgres/extension/functions/perform_reordered_op_installer_function.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/perform_reordered_op_installer_function.sql.eex @@ -1,11 +1,11 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.install_function__perform_reordered_op(schema_name TEXT, table_name TEXT, primary_key_list TEXT[], non_pk_column_list TEXT[]) +CREATE OR REPLACE FUNCTION <%= schema() %>.install_function__perform_reordered_op(schema_name TEXT, table_name TEXT, primary_key_list TEXT[], non_pk_column_list TEXT[]) RETURNS TEXT LANGUAGE PLPGSQL AS $outer_function$ DECLARE function_name TEXT := 'perform_reordered_op___' || schema_name || '__' || table_name; shadow_table_name TEXT := 'shadow__' || schema_name || '__' || table_name; tombstone_table_name TEXT := 'tombstone__' || schema_name || '__' || table_name; - tag_column_list TEXT[] := electric.format_every(non_pk_column_list, '_tag_%s'); + tag_column_list TEXT[] := <%= schema() %>.format_every(non_pk_column_list, '_tag_%s'); reordered_column_list TEXT[]; where_pks_equal_shadow TEXT; built_row_fill_pks TEXT; @@ -20,38 +20,38 @@ DECLARE built_row_overrides TEXT; update_clause TEXT; BEGIN - reordered_column_list := electric.format_every(non_pk_column_list, '__reordered_%s'); + reordered_column_list := <%= schema() %>.format_every(non_pk_column_list, '__reordered_%s'); - where_pks_equal_shadow := electric.format_every_and_join(primary_key_list, '%1$I = shadow_row.%1$I', ' AND '); + where_pks_equal_shadow := <%= schema() %>.format_every_and_join(primary_key_list, '%1$I = shadow_row.%1$I', ' AND '); - built_row_fill_pks := electric.format_every_and_join(primary_key_list, E'\n built_row.%1$I := shadow_row.%1$I;', ''); - built_row_fill_from_reordered := electric.zip_format_every_and_join(non_pk_column_list, reordered_column_list, E'\n built_row.%1$I := shadow_row.%2$I;', ''); + built_row_fill_pks := <%= schema() %>.format_every_and_join(primary_key_list, E'\n built_row.%1$I := shadow_row.%1$I;', ''); + built_row_fill_from_reordered := <%= schema() %>.zip_format_every_and_join(non_pk_column_list, reordered_column_list, E'\n built_row.%1$I := shadow_row.%2$I;', ''); - case_pks := electric.format_every_and_join(primary_key_list, + case_pks := <%= schema() %>.format_every_and_join(primary_key_list, $$ CASE WHEN main.%1$I IS NOT NULL THEN main.%1$I ELSE NULL END as %1$I$$, ','); - all_pks_present_formatter := electric.format_every_and_join(primary_key_list, '%%1$I.%1$I IS NOT NULL', ' AND '); + all_pks_present_formatter := <%= schema() %>.format_every_and_join(primary_key_list, '%%1$I.%1$I IS NOT NULL', ' AND '); case_columns_formatter := format( $$ CASE WHEN %s THEN main.%%1$I WHEN %s THEN tomb.%%1$I END as %%1$I$$, format(all_pks_present_formatter, 'main'), format(all_pks_present_formatter, 'tomb')); - case_columns := electric.format_every_and_join(non_pk_column_list, case_columns_formatter, ','); + case_columns := <%= schema() %>.format_every_and_join(non_pk_column_list, case_columns_formatter, ','); - on_primary_keys := electric.format_every_and_join(primary_key_list, 'main.%1$I = tomb.%1$I', ' AND '); + on_primary_keys := <%= schema() %>.format_every_and_join(primary_key_list, 'main.%1$I = tomb.%1$I', ' AND '); where_pk_main_or_tomb_clause := - '(' || electric.format_every_and_join(primary_key_list, 'main.%1$I = shadow_row.%1$I', ' AND ') + '(' || <%= schema() %>.format_every_and_join(primary_key_list, 'main.%1$I = shadow_row.%1$I', ' AND ') || ') OR (' - || electric.format_every_and_join(primary_key_list, 'tomb.%1$I = shadow_row.%1$I', ' AND ') || ')'; + || <%= schema() %>.format_every_and_join(primary_key_list, 'tomb.%1$I = shadow_row.%1$I', ' AND ') || ')'; - current_row_fill_from_reordered := electric.zip_format_every_and_join(non_pk_column_list, reordered_column_list, E'\n current_row.%1$I = shadow_row.%2$I;', ''); + current_row_fill_from_reordered := <%= schema() %>.zip_format_every_and_join(non_pk_column_list, reordered_column_list, E'\n current_row.%1$I = shadow_row.%2$I;', ''); - built_row_overrides := electric.zip_format_every_and_join(tag_column_list, non_pk_column_list, + built_row_overrides := <%= schema() %>.zip_format_every_and_join(tag_column_list, non_pk_column_list, $$ IF shadow_row.%1$I != shadow_row._tag OR NOT shadow_row._modified_columns_bit_mask[%3$s] THEN built_row.%2$I = current_row.%2$I; @@ -67,7 +67,7 @@ BEGIN $$, format('%I.%I', schema_name, table_name), where_pks_equal_shadow, - electric.format_every_and_join(non_pk_column_list, '%1$I = built_row.%1$I')); + <%= schema() %>.format_every_and_join(non_pk_column_list, '%1$I = built_row.%1$I')); ELSE update_clause := 'NULL;'; -- No-op, since there are no non-pk columns END IF; @@ -75,14 +75,14 @@ BEGIN -- The `%n$I` placeholders use n-th argument for formatting. -- Generally, 1 is a function name, 2 is a shadow table name, 3 is a tombstone table name EXECUTE format($injected$ - CREATE OR REPLACE FUNCTION electric.%1$I(shadow_row electric.%2$I) + CREATE OR REPLACE FUNCTION <%= schema() %>.%1$I(shadow_row <%= schema() %>.%2$I) RETURNS VOID LANGUAGE PLPGSQL SECURITY DEFINER AS $function$ DECLARE built_row %4$s%%ROWTYPE; current_row %4$s%%ROWTYPE; - tombstone_row electric.%3$I%%ROWTYPE; + tombstone_row <%= schema() %>.%3$I%%ROWTYPE; old_row_found boolean; BEGIN RAISE DEBUG ' Preparing a real operation based on shadow row %%', to_json(shadow_row); @@ -110,7 +110,7 @@ BEGIN %8$s INTO current_row FROM %4$s AS main - FULL OUTER JOIN electric.%3$I AS tomb + FULL OUTER JOIN <%= schema() %>.%3$I AS tomb ON %9$s WHERE %10$s; IF NOT FOUND THEN @@ -151,14 +151,14 @@ BEGIN where_pks_equal_shadow, -- 5 built_row_fill_pks, -- 6 built_row_fill_from_reordered, -- 7 - electric.append_string_unless_empty(case_pks, case_columns), -- 8 + <%= schema() %>.append_string_unless_empty(case_pks, case_columns), -- 8 on_primary_keys, -- 9 where_pk_main_or_tomb_clause, -- 10 current_row_fill_from_reordered, -- 11 format(all_pks_present_formatter, 'current_row'), -- 12 built_row_overrides, -- 13 - electric.format_every_and_join(primary_key_list || non_pk_column_list, '%I'), -- 14 - electric.format_every_and_join(primary_key_list || non_pk_column_list, 'built_row.%I'), -- 15 + <%= schema() %>.format_every_and_join(primary_key_list || non_pk_column_list, '%I'), -- 14 + <%= schema() %>.format_every_and_join(primary_key_list || non_pk_column_list, 'built_row.%I'), -- 15 update_clause -- 16 ); diff --git a/components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers/utility_functions.sql b/components/electric/lib/electric/postgres/extension/functions/string_utils.sql.eex similarity index 66% rename from components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers/utility_functions.sql rename to components/electric/lib/electric/postgres/extension/functions/string_utils.sql.eex index 2fc908180e..e32b63fcfd 100644 --- a/components/electric/lib/electric/postgres/extension/migrations/20230512000000_conflict_resolution_triggers/utility_functions.sql +++ b/components/electric/lib/electric/postgres/extension/functions/string_utils.sql.eex @@ -1,5 +1,4 @@ --- Utility function -CREATE OR REPLACE FUNCTION electric.format_every(arr TEXT[], format_pattern TEXT) +CREATE OR REPLACE FUNCTION <%= schema() %>.format_every(arr TEXT[], format_pattern TEXT) RETURNS TEXT[] STABLE PARALLEL SAFE RETURNS NULL ON NULL INPUT @@ -8,7 +7,7 @@ CREATE OR REPLACE FUNCTION electric.format_every(arr TEXT[], format_pattern TEXT $$; -- Utility function -CREATE OR REPLACE FUNCTION electric.format_every_and_join(arr TEXT[], format_pattern TEXT, joiner TEXT DEFAULT ', ') +CREATE OR REPLACE FUNCTION <%= schema() %>.format_every_and_join(arr TEXT[], format_pattern TEXT, joiner TEXT DEFAULT ', ') RETURNS TEXT STABLE PARALLEL SAFE RETURNS NULL ON NULL INPUT @@ -17,7 +16,7 @@ CREATE OR REPLACE FUNCTION electric.format_every_and_join(arr TEXT[], format_pat $$; -- Utility function -CREATE OR REPLACE FUNCTION electric.zip_format_every_and_join(arr1 TEXT[], arr2 TEXT[], format_pattern TEXT, joiner TEXT DEFAULT ', ') +CREATE OR REPLACE FUNCTION <%= schema() %>.zip_format_every_and_join(arr1 TEXT[], arr2 TEXT[], format_pattern TEXT, joiner TEXT DEFAULT ', ') RETURNS TEXT STABLE PARALLEL SAFE RETURNS NULL ON NULL INPUT @@ -25,7 +24,7 @@ CREATE OR REPLACE FUNCTION electric.zip_format_every_and_join(arr1 TEXT[], arr2 SELECT array_to_string(array_agg(format(format_pattern, x, y, ordinality)), joiner) FROM unnest(arr1, arr2) WITH ORDINALITY AS val(x, y, ordinality); $$; -CREATE OR REPLACE FUNCTION electric.append_string_unless_empty(str1 TEXT, str2 TEXT, joiner TEXT DEFAULT ', ') +CREATE OR REPLACE FUNCTION <%= schema() %>.append_string_unless_empty(str1 TEXT, str2 TEXT, joiner TEXT DEFAULT ', ') RETURNS TEXT STABLE PARALLEL SAFE CALLED ON NULL INPUT @@ -37,4 +36,4 @@ CREATE OR REPLACE FUNCTION electric.append_string_unless_empty(str1 TEXT, str2 T RETURN str1; END IF; END - $$; \ No newline at end of file + $$; diff --git a/components/electric/lib/electric/postgres/extension/functions/tx_has_assigned_version.sql.eex b/components/electric/lib/electric/postgres/extension/functions/tx_has_assigned_version.sql.eex index 3ad489b3e2..26dd60ec1f 100644 --- a/components/electric/lib/electric/postgres/extension/functions/tx_has_assigned_version.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/tx_has_assigned_version.sql.eex @@ -1,11 +1,11 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.tx_has_assigned_version() RETURNS bool SECURITY DEFINER AS $function$ +CREATE OR REPLACE FUNCTION <%= schema() %>.tx_has_assigned_version() RETURNS bool SECURITY DEFINER AS $function$ DECLARE - _txid <%= @txid_type %>; - _txts <%= @txts_type %>; + _txid <%= txid_type() %>; + _txts <%= txts_type() %>; _version text; BEGIN - SELECT txid, txts INTO _txid, _txts FROM <%= @schema %>.current_transaction_id(); - SELECT version INTO _version FROM <%= @version_table %> WHERE txid = _txid AND txts = _txts; + SELECT txid, txts INTO _txid, _txts FROM <%= schema() %>.current_transaction_id(); + SELECT version INTO _version FROM <%= version_table() %> WHERE txid = _txid AND txts = _txts; RAISE DEBUG 'transaction %/% has version: %', _txid, _txts, _version; RETURN _version IS NOT NULL; END; diff --git a/components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex b/components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex index 083b427223..12fec8f632 100644 --- a/components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION electric.upsert_acknowledged_client_lsn() +CREATE OR REPLACE FUNCTION <%= schema() %>.upsert_acknowledged_client_lsn() RETURNS TRIGGER LANGUAGE PLPGSQL SECURITY DEFINER @@ -8,7 +8,7 @@ BEGIN RAISE DEBUG ' Given OLD %', to_json(OLD); RAISE DEBUG ' Given NEW %', to_json(NEW); - INSERT INTO electric.acknowledged_client_lsns AS t + INSERT INTO <%= schema() %>.acknowledged_client_lsns AS t VALUES (NEW.client_id, NEW.lsn) ON CONFLICT (client_id) DO UPDATE SET lsn = NEW.lsn From 088868e236739ee7178f67738e9900efe0367888 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Fri, 20 Oct 2023 01:25:46 +0300 Subject: [PATCH 3/9] Disable SQL function definitions in Extension tests --- components/electric/lib/electric/postgres/extension.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/electric/lib/electric/postgres/extension.ex b/components/electric/lib/electric/postgres/extension.ex index 5bcef8ce1d..8f483b6065 100644 --- a/components/electric/lib/electric/postgres/extension.ex +++ b/components/electric/lib/electric/postgres/extension.ex @@ -357,7 +357,9 @@ defmodule Electric.Postgres.Extension do end) end) - :ok = define_functions(txconn) + if module == __MODULE__ do + :ok = define_functions(txconn) + end {:ok, newly_applied_versions} end) From 3b3c8ed654063e2d4c0ebcc5f173943c1d2b3916 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Tue, 31 Oct 2023 00:44:11 +0200 Subject: [PATCH 4/9] Prettify the log output of SQL routine definitions Log sample: 00:43:08.719 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'capture_ddl.sql.eex' 00:43:08.720 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'current_transaction_id.sql.eex' 00:43:08.720 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'current_xact_id.sql.eex' 00:43:08.721 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'current_xact_ts.sql.eex' 00:43:08.723 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'ddlx/assign.sql.eex' 00:43:08.724 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'ddlx/disable.sql.eex' 00:43:08.725 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'ddlx/enable.sql.eex' 00:43:08.725 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'ddlx/grant.sql.eex' 00:43:08.726 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'ddlx/unassign.sql.eex' 00:43:08.727 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'electrify.sql.eex' 00:43:08.728 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'electrify/__validate_table_column_defaults.sql.eex' 00:43:08.728 pid=<0.1285.0> origin=postgres_1 [debug] Successfully (re)defined SQL routine from 'electrify/__validate_table_column_types.sql.eex' 00:45:22.624 pid=<0.1285.0> origin=postgres_1 [error] initialization for postgresql failed with reason: {:rollback, %RuntimeError{message: "Failed to define SQL routine from 'electrify.sql.eex' with error: {:error, {:error, :error, \"42601\", :syntax_error, \"syntax error at or near \\\"SELECD\\\"\", [file: \"scan.l\", line: \"1176\", position: \"243\", routine: \"scanner_yyerror\", severity: \"ERROR\"]}}"}} --- .../electric/lib/electric/postgres/extension.ex | 6 +++--- .../lib/electric/postgres/extension/functions.ex | 14 +++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/components/electric/lib/electric/postgres/extension.ex b/components/electric/lib/electric/postgres/extension.ex index 8f483b6065..88f06f3cb7 100644 --- a/components/electric/lib/electric/postgres/extension.ex +++ b/components/electric/lib/electric/postgres/extension.ex @@ -287,14 +287,14 @@ defmodule Electric.Postgres.Extension do @spec define_functions(conn) :: :ok def define_functions(conn) do - Enum.each(Functions.list(), fn {name, sql} -> + Enum.each(Functions.list(), fn {path, sql} -> conn |> :epgsql.squery(sql) |> List.wrap() |> Enum.find(&(not match?({:ok, [], []}, &1))) |> case do - nil -> Logger.debug("Successfully (re)defined SQL function/procedure '#{name}'") - error -> raise "Failed to define function '#{name}' with error: #{inspect(error)}" + nil -> Logger.debug("Successfully (re)defined SQL routine from '#{path}'") + error -> raise "Failed to define SQL routine from '#{path}' with error: #{inspect(error)}" end end) end diff --git a/components/electric/lib/electric/postgres/extension/functions.ex b/components/electric/lib/electric/postgres/extension/functions.ex index a667e208bf..ff68bad8ee 100644 --- a/components/electric/lib/electric/postgres/extension/functions.ex +++ b/components/electric/lib/electric/postgres/extension/functions.ex @@ -12,16 +12,19 @@ defmodule Electric.Postgres.Extension.Functions do |> Path.expand(__DIR__) |> Path.wildcard() - function_names = + function_paths = for path <- sql_files do @external_resource path + fpath = Path.relative_to(path, Path.expand("./functions", __DIR__)) name = path |> Path.basename(".sql.eex") |> String.to_atom() _ = EEx.function_from_file(:def, name, path, []) - name + {fpath, name} end + function_names = for {_fpath, name} <- function_paths, do: name + fn_name_type = Enum.reduce(function_names, fn name, code -> quote do @@ -31,8 +34,9 @@ defmodule Electric.Postgres.Extension.Functions do @typep name :: unquote(fn_name_type) @typep sql :: String.t() - @type function_list :: [{name, sql}] + @type function_list :: [{binary, sql}] + @function_paths function_paths @function_names function_names @doc """ @@ -44,8 +48,8 @@ defmodule Electric.Postgres.Extension.Functions do # here. See VAX-1016 for details. @spec list :: function_list def list do - for name <- @function_names do - {name, by_name(name)} + for {path, name} <- @function_paths do + {path, by_name(name)} end end From d04e5347312df677848bf733e492489263e79b71 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Sat, 23 Dec 2023 00:16:36 +0200 Subject: [PATCH 5/9] Load SQL function templates on-demand at run time --- .../electric/postgres/extension/functions.ex | 34 +++++++++++-------- .../extension/functions/electrify.sql.eex | 4 +-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/components/electric/lib/electric/postgres/extension/functions.ex b/components/electric/lib/electric/postgres/extension/functions.ex index ff68bad8ee..c98f5bea3a 100644 --- a/components/electric/lib/electric/postgres/extension/functions.ex +++ b/components/electric/lib/electric/postgres/extension/functions.ex @@ -3,7 +3,9 @@ defmodule Electric.Postgres.Extension.Functions do This module organizes SQL functions that are to be defined in Electric's internal database schema. """ - import Electric.Postgres.Extension + # Import all functions from the Extension module to make them available for calling inside SQL function templates when + # those templates are being evaludated by EEx. + import Electric.Postgres.Extension, warn: false require EEx @@ -14,16 +16,12 @@ defmodule Electric.Postgres.Extension.Functions do function_paths = for path <- sql_files do - @external_resource path - - fpath = Path.relative_to(path, Path.expand("./functions", __DIR__)) + relpath = Path.relative_to(path, Path.expand("./functions", __DIR__)) name = path |> Path.basename(".sql.eex") |> String.to_atom() - _ = EEx.function_from_file(:def, name, path, []) - - {fpath, name} + {relpath, name} end - function_names = for {_fpath, name} <- function_paths, do: name + function_names = for {_relpath, name} <- function_paths, do: name fn_name_type = Enum.reduce(function_names, fn name, code -> @@ -33,14 +31,14 @@ defmodule Electric.Postgres.Extension.Functions do end) @typep name :: unquote(fn_name_type) - @typep sql :: String.t() - @type function_list :: [{binary, sql}] + @typep sql :: binary + @type function_list :: [{Path.t(), sql}] @function_paths function_paths @function_names function_names @doc """ - Get a list of `{name, SQL}` pairs where the the SQL code contains the definition of a function (or multiple functions). + Get a list of `{name, SQL}` pairs where the SQL code contains the definition of a function (or multiple functions). Every function in the list is defined as `CREATE OR REPLACE FUNCTION`. """ @@ -48,8 +46,8 @@ defmodule Electric.Postgres.Extension.Functions do # here. See VAX-1016 for details. @spec list :: function_list def list do - for {path, name} <- @function_paths do - {path, by_name(name)} + for {relpath, _name} <- @function_paths do + {relpath, eval_template(relpath)} end end @@ -61,8 +59,14 @@ defmodule Electric.Postgres.Extension.Functions do """ @spec by_name(name) :: sql def by_name(name) when name in @function_names do - apply(__MODULE__, name, []) + {relpath, ^name} = List.keyfind(@function_paths, name, 1) + eval_template(relpath) end - defp publication_sql, do: add_table_to_publication_sql("%I.%I") + defp eval_template(relpath) do + Path.join([Path.expand(__DIR__), "functions", relpath]) + |> EEx.compile_file() + |> Code.eval_quoted([], __ENV__) + |> elem(0) + end end diff --git a/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex b/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex index 540e481f93..fc7650f41e 100644 --- a/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex +++ b/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex @@ -44,7 +44,7 @@ BEGIN VALUES (_schema, _table, _oid); - EXECUTE format('<%= publication_sql() %>;', _schema, _table); + EXECUTE format('<%= add_table_to_publication_sql("%I.%I") %>;', _schema, _table); -- We want to disable any possible hooks from `CREATE TABLE` statements PERFORM set_config('<%= schema() %>.is_in_event_trigger', 'true', true); @@ -52,7 +52,7 @@ BEGIN PERFORM set_config('<%= schema() %>.is_in_event_trigger', '', true); EXECUTE format( - '<%= publication_sql() %>;', + '<%= add_table_to_publication_sql("%I.%I") %>;', '<%= schema() %>', ('shadow__' || _schema || '__' || _table)::name ); From ebd078da3da5321a0497ed55aace0446df42baf5 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Sat, 23 Dec 2023 00:37:50 +0200 Subject: [PATCH 6/9] Move SQL function templates into priv/ ...so that they are available in the packaged Elixir release to be loaded and evaluated at run time. --- components/electric/Dockerfile | 3 ++- .../lib/electric/postgres/extension/functions.ex | 14 +++++++------- .../sql_function_templates}/__pg_version.sql.eex | 0 .../__primary_key_list.sql.eex | 0 .../__resolve_table_from_names.sql.eex | 0 .../__session_replication_role.sql.eex | 0 .../sql_function_templates}/__table_schema.sql.eex | 0 .../alter_shadow_table.sql.eex | 0 .../assign_default_version.sql.eex | 0 .../assign_migration_version.sql.eex | 0 .../sql_function_templates}/capture_ddl.sql.eex | 0 .../current_transaction_id.sql.eex | 0 .../current_xact_id.sql.eex | 0 .../current_xact_ts.sql.eex | 0 .../sql_function_templates}/ddlx/assign.sql.eex | 0 .../sql_function_templates}/ddlx/disable.sql.eex | 0 .../sql_function_templates}/ddlx/enable.sql.eex | 0 .../sql_function_templates}/ddlx/grant.sql.eex | 0 .../sql_function_templates}/ddlx/unassign.sql.eex | 0 .../sql_function_templates}/electrify.sql.eex | 0 .../__validate_table_column_defaults.sql.eex | 0 .../__validate_table_column_types.sql.eex | 0 .../electrify/generate_electrified_sql.sql.eex | 0 .../find_fk_to_table.sql.eex | 0 .../install_functions_and_triggers.sql.eex | 0 .../migration_version.sql.eex | 0 ...perform_reordered_op_installer_function.sql.eex | 0 .../sql_function_templates}/string_utils.sql.eex | 0 .../tx_has_assigned_version.sql.eex | 0 .../upsert_acknowledged_client_lsn.sql.eex | 0 30 files changed, 9 insertions(+), 8 deletions(-) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/__pg_version.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/__primary_key_list.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/__resolve_table_from_names.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/__session_replication_role.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/__table_schema.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/alter_shadow_table.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/assign_default_version.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/assign_migration_version.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/capture_ddl.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/current_transaction_id.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/current_xact_id.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/current_xact_ts.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/ddlx/assign.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/ddlx/disable.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/ddlx/enable.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/ddlx/grant.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/ddlx/unassign.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/electrify.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/electrify/__validate_table_column_defaults.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/electrify/__validate_table_column_types.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/electrify/generate_electrified_sql.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/find_fk_to_table.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/install_functions_and_triggers.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/migration_version.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/perform_reordered_op_installer_function.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/string_utils.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/tx_has_assigned_version.sql.eex (100%) rename components/electric/{lib/electric/postgres/extension/functions => priv/sql_function_templates}/upsert_acknowledged_client_lsn.sql.eex (100%) diff --git a/components/electric/Dockerfile b/components/electric/Dockerfile index a7d86a6728..eae25627cf 100644 --- a/components/electric/Dockerfile +++ b/components/electric/Dockerfile @@ -24,9 +24,10 @@ COPY mix.* /app/ RUN mix deps.get RUN mix deps.compile +COPY config/*runtime.exs /app/config/ COPY lib /app/lib/ +COPY priv /app/priv COPY src /app/src/ -COPY config/*runtime.exs /app/config/ ARG ELECTRIC_VERSION=local ARG MAKE_RELEASE_TASK=release diff --git a/components/electric/lib/electric/postgres/extension/functions.ex b/components/electric/lib/electric/postgres/extension/functions.ex index c98f5bea3a..e3cd8023c6 100644 --- a/components/electric/lib/electric/postgres/extension/functions.ex +++ b/components/electric/lib/electric/postgres/extension/functions.ex @@ -9,14 +9,14 @@ defmodule Electric.Postgres.Extension.Functions do require EEx - sql_files = - "functions/**/*.sql.eex" - |> Path.expand(__DIR__) - |> Path.wildcard() + @template_dir "priv/sql_function_templates" + + template_dir_path = Application.app_dir(:electric, @template_dir) + sql_template_paths = Path.wildcard(template_dir_path <> "/**/*.sql.eex") function_paths = - for path <- sql_files do - relpath = Path.relative_to(path, Path.expand("./functions", __DIR__)) + for path <- sql_template_paths do + relpath = Path.relative_to(path, template_dir_path) name = path |> Path.basename(".sql.eex") |> String.to_atom() {relpath, name} end @@ -64,7 +64,7 @@ defmodule Electric.Postgres.Extension.Functions do end defp eval_template(relpath) do - Path.join([Path.expand(__DIR__), "functions", relpath]) + Path.join(Application.app_dir(:electric, @template_dir), relpath) |> EEx.compile_file() |> Code.eval_quoted([], __ENV__) |> elem(0) diff --git a/components/electric/lib/electric/postgres/extension/functions/__pg_version.sql.eex b/components/electric/priv/sql_function_templates/__pg_version.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/__pg_version.sql.eex rename to components/electric/priv/sql_function_templates/__pg_version.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/__primary_key_list.sql.eex b/components/electric/priv/sql_function_templates/__primary_key_list.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/__primary_key_list.sql.eex rename to components/electric/priv/sql_function_templates/__primary_key_list.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/__resolve_table_from_names.sql.eex b/components/electric/priv/sql_function_templates/__resolve_table_from_names.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/__resolve_table_from_names.sql.eex rename to components/electric/priv/sql_function_templates/__resolve_table_from_names.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/__session_replication_role.sql.eex b/components/electric/priv/sql_function_templates/__session_replication_role.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/__session_replication_role.sql.eex rename to components/electric/priv/sql_function_templates/__session_replication_role.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/__table_schema.sql.eex b/components/electric/priv/sql_function_templates/__table_schema.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/__table_schema.sql.eex rename to components/electric/priv/sql_function_templates/__table_schema.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/alter_shadow_table.sql.eex b/components/electric/priv/sql_function_templates/alter_shadow_table.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/alter_shadow_table.sql.eex rename to components/electric/priv/sql_function_templates/alter_shadow_table.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/assign_default_version.sql.eex b/components/electric/priv/sql_function_templates/assign_default_version.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/assign_default_version.sql.eex rename to components/electric/priv/sql_function_templates/assign_default_version.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/assign_migration_version.sql.eex b/components/electric/priv/sql_function_templates/assign_migration_version.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/assign_migration_version.sql.eex rename to components/electric/priv/sql_function_templates/assign_migration_version.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/capture_ddl.sql.eex b/components/electric/priv/sql_function_templates/capture_ddl.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/capture_ddl.sql.eex rename to components/electric/priv/sql_function_templates/capture_ddl.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/current_transaction_id.sql.eex b/components/electric/priv/sql_function_templates/current_transaction_id.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/current_transaction_id.sql.eex rename to components/electric/priv/sql_function_templates/current_transaction_id.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/current_xact_id.sql.eex b/components/electric/priv/sql_function_templates/current_xact_id.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/current_xact_id.sql.eex rename to components/electric/priv/sql_function_templates/current_xact_id.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/current_xact_ts.sql.eex b/components/electric/priv/sql_function_templates/current_xact_ts.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/current_xact_ts.sql.eex rename to components/electric/priv/sql_function_templates/current_xact_ts.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/assign.sql.eex b/components/electric/priv/sql_function_templates/ddlx/assign.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/ddlx/assign.sql.eex rename to components/electric/priv/sql_function_templates/ddlx/assign.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/disable.sql.eex b/components/electric/priv/sql_function_templates/ddlx/disable.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/ddlx/disable.sql.eex rename to components/electric/priv/sql_function_templates/ddlx/disable.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/enable.sql.eex b/components/electric/priv/sql_function_templates/ddlx/enable.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/ddlx/enable.sql.eex rename to components/electric/priv/sql_function_templates/ddlx/enable.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/grant.sql.eex b/components/electric/priv/sql_function_templates/ddlx/grant.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/ddlx/grant.sql.eex rename to components/electric/priv/sql_function_templates/ddlx/grant.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/ddlx/unassign.sql.eex b/components/electric/priv/sql_function_templates/ddlx/unassign.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/ddlx/unassign.sql.eex rename to components/electric/priv/sql_function_templates/ddlx/unassign.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex b/components/electric/priv/sql_function_templates/electrify.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/electrify.sql.eex rename to components/electric/priv/sql_function_templates/electrify.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_defaults.sql.eex b/components/electric/priv/sql_function_templates/electrify/__validate_table_column_defaults.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_defaults.sql.eex rename to components/electric/priv/sql_function_templates/electrify/__validate_table_column_defaults.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_types.sql.eex b/components/electric/priv/sql_function_templates/electrify/__validate_table_column_types.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/electrify/__validate_table_column_types.sql.eex rename to components/electric/priv/sql_function_templates/electrify/__validate_table_column_types.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/electrify/generate_electrified_sql.sql.eex b/components/electric/priv/sql_function_templates/electrify/generate_electrified_sql.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/electrify/generate_electrified_sql.sql.eex rename to components/electric/priv/sql_function_templates/electrify/generate_electrified_sql.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/find_fk_to_table.sql.eex b/components/electric/priv/sql_function_templates/find_fk_to_table.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/find_fk_to_table.sql.eex rename to components/electric/priv/sql_function_templates/find_fk_to_table.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/install_functions_and_triggers.sql.eex b/components/electric/priv/sql_function_templates/install_functions_and_triggers.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/install_functions_and_triggers.sql.eex rename to components/electric/priv/sql_function_templates/install_functions_and_triggers.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/migration_version.sql.eex b/components/electric/priv/sql_function_templates/migration_version.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/migration_version.sql.eex rename to components/electric/priv/sql_function_templates/migration_version.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/perform_reordered_op_installer_function.sql.eex b/components/electric/priv/sql_function_templates/perform_reordered_op_installer_function.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/perform_reordered_op_installer_function.sql.eex rename to components/electric/priv/sql_function_templates/perform_reordered_op_installer_function.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/string_utils.sql.eex b/components/electric/priv/sql_function_templates/string_utils.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/string_utils.sql.eex rename to components/electric/priv/sql_function_templates/string_utils.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/tx_has_assigned_version.sql.eex b/components/electric/priv/sql_function_templates/tx_has_assigned_version.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/tx_has_assigned_version.sql.eex rename to components/electric/priv/sql_function_templates/tx_has_assigned_version.sql.eex diff --git a/components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex b/components/electric/priv/sql_function_templates/upsert_acknowledged_client_lsn.sql.eex similarity index 100% rename from components/electric/lib/electric/postgres/extension/functions/upsert_acknowledged_client_lsn.sql.eex rename to components/electric/priv/sql_function_templates/upsert_acknowledged_client_lsn.sql.eex From 1b10be844dacea3341859dc9d1ba628ca65145d8 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Sat, 23 Dec 2023 00:45:24 +0200 Subject: [PATCH 7/9] Remove empty lines from SQL function templates --- .../sql_function_templates/__resolve_table_from_names.sql.eex | 1 - .../electric/priv/sql_function_templates/__table_schema.sql.eex | 1 - .../priv/sql_function_templates/assign_migration_version.sql.eex | 1 - .../electric/priv/sql_function_templates/capture_ddl.sql.eex | 1 - .../priv/sql_function_templates/current_transaction_id.sql.eex | 1 - .../priv/sql_function_templates/migration_version.sql.eex | 1 - 6 files changed, 6 deletions(-) diff --git a/components/electric/priv/sql_function_templates/__resolve_table_from_names.sql.eex b/components/electric/priv/sql_function_templates/__resolve_table_from_names.sql.eex index 6f020290e5..52a5026ed5 100644 --- a/components/electric/priv/sql_function_templates/__resolve_table_from_names.sql.eex +++ b/components/electric/priv/sql_function_templates/__resolve_table_from_names.sql.eex @@ -1,4 +1,3 @@ - CREATE OR REPLACE FUNCTION <%= schema() %>.__resolve_table_from_names( name1 text, name2 text, diff --git a/components/electric/priv/sql_function_templates/__table_schema.sql.eex b/components/electric/priv/sql_function_templates/__table_schema.sql.eex index 1e2dc721a2..a959f47c64 100644 --- a/components/electric/priv/sql_function_templates/__table_schema.sql.eex +++ b/components/electric/priv/sql_function_templates/__table_schema.sql.eex @@ -1,4 +1,3 @@ - CREATE OR REPLACE FUNCTION <%= schema() %>.__table_schema(table_oid oid) RETURNS name SECURITY DEFINER AS $function$ DECLARE _schema name; diff --git a/components/electric/priv/sql_function_templates/assign_migration_version.sql.eex b/components/electric/priv/sql_function_templates/assign_migration_version.sql.eex index c0ccfcd934..0124f7647e 100644 --- a/components/electric/priv/sql_function_templates/assign_migration_version.sql.eex +++ b/components/electric/priv/sql_function_templates/assign_migration_version.sql.eex @@ -1,4 +1,3 @@ - CREATE OR REPLACE PROCEDURE <%= schema() %>.assign_migration_version(_version text, _priority int) SECURITY DEFINER AS $function$ BEGIN -- if there's already a version for this tx, then overwrite it only if the priority of diff --git a/components/electric/priv/sql_function_templates/capture_ddl.sql.eex b/components/electric/priv/sql_function_templates/capture_ddl.sql.eex index 885cc831ad..2b54e54ea8 100644 --- a/components/electric/priv/sql_function_templates/capture_ddl.sql.eex +++ b/components/electric/priv/sql_function_templates/capture_ddl.sql.eex @@ -1,4 +1,3 @@ - CREATE OR REPLACE PROCEDURE <%= schema() %>.capture_ddl(_query text DEFAULT NULL) SECURITY DEFINER AS $function$ BEGIN diff --git a/components/electric/priv/sql_function_templates/current_transaction_id.sql.eex b/components/electric/priv/sql_function_templates/current_transaction_id.sql.eex index a49535fbe8..c4d46fb4cd 100644 --- a/components/electric/priv/sql_function_templates/current_transaction_id.sql.eex +++ b/components/electric/priv/sql_function_templates/current_transaction_id.sql.eex @@ -1,4 +1,3 @@ - CREATE OR REPLACE FUNCTION <%= schema() %>.current_transaction_id( OUT txid <%= txid_type() %>, OUT txts <%= txts_type() %> diff --git a/components/electric/priv/sql_function_templates/migration_version.sql.eex b/components/electric/priv/sql_function_templates/migration_version.sql.eex index 42ba63217c..595c30734e 100644 --- a/components/electric/priv/sql_function_templates/migration_version.sql.eex +++ b/components/electric/priv/sql_function_templates/migration_version.sql.eex @@ -1,4 +1,3 @@ - CREATE OR REPLACE PROCEDURE <%= schema() %>.migration_version(_version text) SECURITY DEFINER AS $function$ BEGIN -- this is the user-facing api, so the priority of the version is set to the highest possible value From e5503ea748bc53998eedd8364932c316e4e92dc6 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Mon, 8 Jan 2024 16:05:39 +0200 Subject: [PATCH 8/9] Improve compilation errors for SQL function templates --- .../electric/lib/electric/postgres/extension/functions.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/components/electric/lib/electric/postgres/extension/functions.ex b/components/electric/lib/electric/postgres/extension/functions.ex index e3cd8023c6..c5bbe78b44 100644 --- a/components/electric/lib/electric/postgres/extension/functions.ex +++ b/components/electric/lib/electric/postgres/extension/functions.ex @@ -64,9 +64,13 @@ defmodule Electric.Postgres.Extension.Functions do end defp eval_template(relpath) do + # This hack is necessary to get a meaningful error when EEx fails to evaluate the template. + # Without it, errors will say that it is lib/electric/postgres/extension/functions.ex that failed to compile. + env = %{__ENV__ | file: relpath} + Path.join(Application.app_dir(:electric, @template_dir), relpath) - |> EEx.compile_file() - |> Code.eval_quoted([], __ENV__) + |> EEx.compile_file(file: relpath) + |> Code.eval_quoted([], env) |> elem(0) end end From 1f1bf0a08f87e79b164ffc0ae3fcd3e12c0668e1 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Mon, 8 Jan 2024 16:06:48 +0200 Subject: [PATCH 9/9] fixup! Move SQL function templates into priv/ --- .../electrify/generate_electrified_sql.sql.eex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/electric/priv/sql_function_templates/electrify/generate_electrified_sql.sql.eex b/components/electric/priv/sql_function_templates/electrify/generate_electrified_sql.sql.eex index 3d10885fbd..f7cb384ab1 100644 --- a/components/electric/priv/sql_function_templates/electrify/generate_electrified_sql.sql.eex +++ b/components/electric/priv/sql_function_templates/electrify/generate_electrified_sql.sql.eex @@ -1,4 +1,4 @@ -CREATE OR REPLACE FUNCTION <%= @schema %>.generate_electrified_sql(table_oid regclass) +CREATE OR REPLACE FUNCTION <%= schema() %>.generate_electrified_sql(table_oid regclass) RETURNS text SECURITY DEFINER AS $function$ @@ -14,11 +14,11 @@ BEGIN WHERE attrelid = table_oid AND attnum > 0 AND NOT attisdropped AND typtype = 'e' ORDER BY attnum LOOP - SELECT <%= @schema %>.ddlgen_create(_col_type) INTO _ddlgen_sql; + SELECT <%= schema() %>.ddlgen_create(_col_type) INTO _ddlgen_sql; _result_sql := _result_sql || _ddlgen_sql; END LOOP; - SELECT <%= @schema %>.ddlgen_create(table_oid) INTO _ddlgen_sql; + SELECT <%= schema() %>.ddlgen_create(table_oid) INTO _ddlgen_sql; _result_sql := _result_sql || _ddlgen_sql; RETURN _result_sql;