From 4093f05054ad1ad0ecf10663957e86f1b7a812d5 Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Thu, 4 Nov 2021 19:13:34 +1100 Subject: [PATCH 1/9] fix tests --- tests/test_main.py | 153 +++++++++++++++++++++++++++++++-------------- 1 file changed, 105 insertions(+), 48 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 8fd90aa9..10605fe6 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,70 +3,127 @@ import pytest import unittest.mock as mock import schemachange.cli +import tempfile +from textwrap import dedent +DEFAULT_CONFIG = { + 'root-folder': os.path.abspath('.'), + 'modules-folder': None, + 'snowflake-account': None, + 'snowflake-user': None, + 'snowflake-role':None, + 'snowflake-warehouse': None, + 'snowflake-database': None, + 'change-history-table': None, + 'vars': {}, + 'create-change-history-table': False, + 'autocommit': False, + 'verbose': False, + 'dry-run': False, +} @pytest.mark.parametrize("args, expected", [ - (["schemachange"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "--config-folder", "test"], ('test', None, None, None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "-f", '.'], ('.', '.', None, None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "--modules-folder", "modules-folder"], ('.', None, "modules-folder", None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "--snowflake-account", "account"], ('.', None, None, "account", None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "--snowflake-user", "user"], ('.', None, None, None, "user", None, None, None, None, None, False, False, False, False)), - (["schemachange", "--snowflake-role", "role"], ('.', None, None, None, None, "role", None, None, None, None, False, False, False, False)), - (["schemachange", "--snowflake-warehouse", "warehouse"], ('.', None, None, None, None, None, "warehouse", None, None, None, False, False, False, False)), - (["schemachange", "--snowflake-database", "database"], ('.', None, None, None, None, None, None, "database", None, None, False, False, False, False)), - (["schemachange", "--change-history-table", "db.schema.table"], ('.', None, None, None, None, None, None, None, "db.schema.table", None, False, False, False, False)), - (["schemachange", "--vars", '{"var1": "val"}'], ('.', None, None, None, None, None, None, None, None, {'var1' : 'val'}, False, False, False, False)), - (["schemachange", "--create-change-history-table"], ('.', None, None, None, None, None, None, None, None, None, True, False, False, False)), - (["schemachange", "--autocommit"], ('.', None, None, None, None, None, None, None, None, None, False, True, False, False)), - (["schemachange", "--verbose"], ('.', None, None, None, None, None, None, None, None, None, False, False, True, False)), - (["schemachange", "--dry-run"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, True)) + (["schemachange"], DEFAULT_CONFIG), + (["schemachange", "deploy"], DEFAULT_CONFIG), + (["schemachange", "deploy", "-f", '.'], + {**DEFAULT_CONFIG, 'root-folder':os.path.abspath('.')}), + (["schemachange", "deploy", "--snowflake-account", "account"], + {**DEFAULT_CONFIG, 'snowflake-account': 'account'}), + (["schemachange", "deploy", "--snowflake-user", "user"], + {**DEFAULT_CONFIG, 'snowflake-user': 'user'}), + (["schemachange", "deploy", "--snowflake-role", "role"], + {**DEFAULT_CONFIG, 'snowflake-role': 'role'}), + (["schemachange", "deploy", "--snowflake-warehouse", "warehouse"], + {**DEFAULT_CONFIG, 'snowflake-warehouse': 'warehouse'}), + (["schemachange", "deploy", "--snowflake-database", "database"], + {**DEFAULT_CONFIG, 'snowflake-database': 'database'}), + (["schemachange", "deploy", "--change-history-table", "db.schema.table"], + {**DEFAULT_CONFIG, 'change-history-table': 'db.schema.table'}), + (["schemachange", "deploy", "--vars", '{"var1": "val"}'], + {**DEFAULT_CONFIG, 'vars': {'var1' : 'val'},}), + (["schemachange", "deploy", "--create-change-history-table"], + {**DEFAULT_CONFIG, 'create-change-history-table': True}), + (["schemachange", "deploy", "--autocommit"], + {**DEFAULT_CONFIG, 'autocommit': True}), + (["schemachange", "deploy", "--verbose"], + {**DEFAULT_CONFIG, 'verbose': True}), + (["schemachange", "deploy", "--dry-run"], + {**DEFAULT_CONFIG, 'dry-run': True}), ]) -def test_main_no_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): + +def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): sys.argv = args - with mock.patch("schemachange.cli.schemachange") as mock_schemachange: + with mock.patch("schemachange.cli.deploy_command") as mock_deploy_command: schemachange.cli.main() - mock_schemachange.assert_called_once_with(*expected) + mock_deploy_command.assert_called_once() + [config,], _call_kwargs = mock_deploy_command.call_args + assert config == expected + +@pytest.mark.parametrize("args, to_mock, expected_args", [ + (["schemachange", "deploy", "--config-folder", "DUMMY"], + "schemachange.cli.deploy_command", + ({**DEFAULT_CONFIG, 'snowflake-account': 'account'},)), + (["schemachange", "render", "script.sql", "--config-folder", "DUMMY"], + "schemachange.cli.render_command", + ({**DEFAULT_CONFIG, 'snowflake-account': 'account'}, "script.sql")) +]) +def test_main_deploy_config_folder(args, to_mock, expected_args): + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, 'schemachange-config.yml'), 'wt') as f: + f.write(dedent(''' + snowflake-account: account + ''')) + args[args.index("DUMMY")] = d + sys.argv = args -@pytest.mark.parametrize("args, expected", [ - (["schemachange", "deploy"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "deploy", "--config-folder", "test"], ('test', None, None, None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "deploy", "-f", '.'], ('.', '.', None, None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "deploy", "--modules-folder", "modules-folder"], ('.', None, "modules-folder", None, None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "deploy", "--snowflake-account", "account"], ('.', None, None, "account", None, None, None, None, None, None, False, False, False, False)), - (["schemachange", "deploy", "--snowflake-user", "user"], ('.', None, None, None, "user", None, None, None, None, None, False, False, False, False)), - (["schemachange", "deploy", "--snowflake-role", "role"], ('.', None, None, None, None, "role", None, None, None, None, False, False, False, False)), - (["schemachange", "deploy", "--snowflake-warehouse", "warehouse"], ('.', None, None, None, None, None, "warehouse", None, None, None, False, False, False, False)), - (["schemachange", "deploy", "--snowflake-database", "database"], ('.', None, None, None, None, None, None, "database", None, None, False, False, False, False)), - (["schemachange", "deploy", "--change-history-table", "db.schema.table"], ('.', None, None, None, None, None, None, None, "db.schema.table", None, False, False, False, False)), - (["schemachange", "deploy", "--vars", '{"var1": "val"}'], ('.', None, None, None, None, None, None, None, None, {'var1' : 'val'}, False, False, False, False)), - (["schemachange", "deploy", "--create-change-history-table"], ('.', None, None, None, None, None, None, None, None, None, True, False, False, False)), - (["schemachange", "deploy", "--autocommit"], ('.', None, None, None, None, None, None, None, None, None, False, True, False, False)), - (["schemachange", "deploy", "--verbose"], ('.', None, None, None, None, None, None, None, None, None, False, False, True, False)), - (["schemachange", "deploy", "--dry-run"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, True)) - + with mock.patch(to_mock) as mock_command: + schemachange.cli.main() + mock_command.assert_called_once() + call_args, _call_kwargs = mock_command.call_args + assert call_args == expected_args + + +@pytest.mark.parametrize("args, to_mock, expected_args", [ + (["schemachange", "deploy", "--modules-folder", "DUMMY"], + "schemachange.cli.deploy_command", + ({**DEFAULT_CONFIG, 'modules-folder': 'DUMMY'},)), + (["schemachange", "render", "script.sql", "--modules-folder", "DUMMY"], + "schemachange.cli.render_command", + ({**DEFAULT_CONFIG, 'modules-folder': 'DUMMY'}, "script.sql")) ]) -def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): - sys.argv = args - - with mock.patch("schemachange.cli.schemachange") as mock_schemachange: - schemachange.cli.main() - mock_schemachange.assert_called_once_with(*expected) +def test_main_deploy_modules_folder(args, to_mock, expected_args): + with tempfile.TemporaryDirectory() as d: + + args[args.index("DUMMY")] = d + expected_args[0]['modules-folder'] = d + sys.argv = args + with mock.patch(to_mock) as mock_command: + schemachange.cli.main() + mock_command.assert_called_once() + call_args, _call_kwargs = mock_command.call_args + assert call_args == expected_args @pytest.mark.parametrize("args, expected", [ - (["schemachange", "render", "script.sql"], ('.', None, None, None, False, "script.sql")), - (["schemachange", "render", "--config-folder", "test", "script.sql"], ("test", None, None, None, False, "script.sql")), - (["schemachange", "render", "--root-folder", '.', "script.sql"], ('.', ".", None, None, False, "script.sql")), - (["schemachange", "render", "--modules-folder", "modules-folder", "script.sql"], ('.', None, "modules-folder", None, False, "script.sql")), - (["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"], ('.', None, None, {'var1' : 'val'}, False, "script.sql")), - (["schemachange", "render", "--verbose", "script.sql"], ('.', None, None, None, True, "script.sql")), + (["schemachange", "render", "script.sql"], + ({**DEFAULT_CONFIG}, "script.sql")), + # (["schemachange", "render", "--config-folder", "test", "script.sql"], + # ("test", None, None, None, False, "script.sql")), + (["schemachange", "render", "--root-folder", '.', "script.sql"], + ({**DEFAULT_CONFIG, 'root-folder': os.path.abspath('.')}, "script.sql")), + #(["schemachange", "render", "--modules-folder", "modules-folder", "script.sql"], ('.', None, "modules-folder", None, False, "script.sql")), + (["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"], + ({**DEFAULT_CONFIG, 'vars': {"var1": "val"}}, "script.sql")), + (["schemachange", "render", "--verbose", "script.sql"], + ({**DEFAULT_CONFIG, 'verbose': True}, "script.sql")), ]) def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): sys.argv = args with mock.patch("schemachange.cli.render_command") as mock_render_command: schemachange.cli.main() - mock_render_command.assert_called_once_with(*expected) + mock_render_command.assert_called_once() + call_args, _call_kwargs = mock_render_command.call_args + assert call_args == expected From 58474c72ddb0b6d60caf59a5780e5b2ccec9bb54 Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Thu, 4 Nov 2021 19:16:16 +1100 Subject: [PATCH 2/9] add github CI --- .github/workflows/test.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..c7ef2a6e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Python tests + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install -e .[dev] + - name: Test with pytest + run: | + pytest \ No newline at end of file From 2030e320e17c6a30b1fabcc7d31874de6822c29f Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Thu, 4 Nov 2021 19:25:06 +1100 Subject: [PATCH 3/9] fixes --- tests/test_main.py | 51 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 10605fe6..68f84215 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -2,7 +2,7 @@ import sys import pytest import unittest.mock as mock -import schemachange.cli +import schemachange.cli import tempfile from textwrap import dedent @@ -22,6 +22,7 @@ 'dry-run': False, } + @pytest.mark.parametrize("args, expected", [ (["schemachange"], DEFAULT_CONFIG), (["schemachange", "deploy"], DEFAULT_CONFIG), @@ -50,16 +51,36 @@ (["schemachange", "deploy", "--dry-run"], {**DEFAULT_CONFIG, 'dry-run': True}), ]) - def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): sys.argv = args - + with mock.patch("schemachange.cli.deploy_command") as mock_deploy_command: schemachange.cli.main() mock_deploy_command.assert_called_once() [config,], _call_kwargs = mock_deploy_command.call_args assert config == expected + +@pytest.mark.parametrize("args, expected", [ + (["schemachange", "render", "script.sql"], + ({**DEFAULT_CONFIG}, "script.sql")), + (["schemachange", "render", "--root-folder", '.', "script.sql"], + ({**DEFAULT_CONFIG, 'root-folder': os.path.abspath('.')}, "script.sql")), + (["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"], + ({**DEFAULT_CONFIG, 'vars': {"var1": "val"}}, "script.sql")), + (["schemachange", "render", "--verbose", "script.sql"], + ({**DEFAULT_CONFIG, 'verbose': True}, "script.sql")), +]) +def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): + sys.argv = args + + with mock.patch("schemachange.cli.render_command") as mock_render_command: + schemachange.cli.main() + mock_render_command.assert_called_once() + call_args, _call_kwargs = mock_render_command.call_args + assert call_args == expected + + @pytest.mark.parametrize("args, to_mock, expected_args", [ (["schemachange", "deploy", "--config-folder", "DUMMY"], "schemachange.cli.deploy_command", @@ -104,26 +125,4 @@ def test_main_deploy_modules_folder(args, to_mock, expected_args): schemachange.cli.main() mock_command.assert_called_once() call_args, _call_kwargs = mock_command.call_args - assert call_args == expected_args - -@pytest.mark.parametrize("args, expected", [ - (["schemachange", "render", "script.sql"], - ({**DEFAULT_CONFIG}, "script.sql")), - # (["schemachange", "render", "--config-folder", "test", "script.sql"], - # ("test", None, None, None, False, "script.sql")), - (["schemachange", "render", "--root-folder", '.', "script.sql"], - ({**DEFAULT_CONFIG, 'root-folder': os.path.abspath('.')}, "script.sql")), - #(["schemachange", "render", "--modules-folder", "modules-folder", "script.sql"], ('.', None, "modules-folder", None, False, "script.sql")), - (["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"], - ({**DEFAULT_CONFIG, 'vars': {"var1": "val"}}, "script.sql")), - (["schemachange", "render", "--verbose", "script.sql"], - ({**DEFAULT_CONFIG, 'verbose': True}, "script.sql")), -]) -def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): - sys.argv = args - - with mock.patch("schemachange.cli.render_command") as mock_render_command: - schemachange.cli.main() - mock_render_command.assert_called_once() - call_args, _call_kwargs = mock_render_command.call_args - assert call_args == expected + assert call_args == expected_args \ No newline at end of file From 93261fd5cc4d9911cd331f8b823076bc321041ac Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Thu, 4 Nov 2021 19:34:05 +1100 Subject: [PATCH 4/9] add windows runner --- .github/workflows/test.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c7ef2a6e..87c33736 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,10 +5,12 @@ on: [push] jobs: build: - runs-on: ubuntu-latest strategy: matrix: python-version: [3.7, 3.8, 3.9] + github-runner: ['ubuntu-latest', 'windows-latest'] + + runs-on: ${{ matrix.github-runner }} steps: - uses: actions/checkout@v2 From 541803202b181b1fa11b0720dde0852067a9b634 Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Mon, 27 Sep 2021 14:27:32 +1000 Subject: [PATCH 5/9] add --explain-first option --- schemachange/cli.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 189aa3c9..92f46ada 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -17,6 +17,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric import dsa from cryptography.hazmat.primitives import serialization +import io # Set a few global variables here _schemachange_version = '3.2.0' @@ -164,6 +165,9 @@ def deploy_command(config): scripts_skipped += 1 continue + if config['explain-first']: + explain_change_script(script, config['vars'], config['snowflake-database'], snowflake_session_parameters, config['autocommit'], config['verbose']) + print("Applying change script %s" % script['script_name']) if not config['dry-run']: apply_change_script(script, content, config['vars'], config['snowflake-database'], change_history_table, snowflake_session_parameters, config['autocommit'], config['verbose']) @@ -200,7 +204,7 @@ def get_alphanum_key(key): def sorted_alphanumeric(data): return sorted(data, key=get_alphanum_key) -def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, change_history_table_override, vars, create_change_history_table, autocommit, verbose, dry_run): +def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, change_history_table_override, vars, create_change_history_table, autocommit, verbose, dry_run, explain_first): config = dict() # First read in the yaml config file, if present @@ -294,6 +298,11 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf if "schemachange" in config['vars']: raise ValueError("The variable schemachange has been reserved for use by schemachange, please use a different name") + if explain_first: + config['explain-first'] = explain_first + if 'explain-first' not in config: + config['explain-first'] = False + return config def get_all_scripts_recursively(root_directory, verbose): @@ -548,6 +557,20 @@ def apply_change_script(script, script_content, vars, default_database, change_h execute_snowflake_query(change_history_table['database_name'], query, snowflake_session_parameters, autocommit, verbose) +def explain_change_script(script, vars, default_database, snowflake_session_parameters, verbose): + ''' + Run "explain " for every in a script of ; ; ... + This will throw an error if the explain fails, which will catch many issues with the script without needing to directly execute it. + ''' + content = get_script_contents_with_variable_replacement(script['script_full_path'], vars, verbose) + + content_io = io.StringIO(content) + statements = snowflake.connector.util_text.split_statements(content_io) + for s, _ in statements: + explain = f"explain {s}" + execute_snowflake_query(default_database, explain, snowflake_session_parameters, False, verbose) + + def main(): parser = argparse.ArgumentParser(prog = 'schemachange', description = 'Apply schema changes to a Snowflake account. Full readme at https://github.com/Snowflake-Labs/schemachange', formatter_class = argparse.RawTextHelpFormatter) subcommands = parser.add_subparsers(dest='subcommand') @@ -567,6 +590,7 @@ def main(): parser_deploy.add_argument('-ac', '--autocommit', action='store_true', help = 'Enable autocommit feature for DML commands (the default is False)', required = False) parser_deploy.add_argument('-v','--verbose', action='store_true', help = 'Display verbose debugging details during execution (the default is False)', required = False) parser_deploy.add_argument('--dry-run', action='store_true', help = 'Run schemachange in dry run mode (the default is False)', required = False) + parser_deploy.add_argument('--explain-first', action='store_true', help = 'Run explain before each query is run properly, which can be used to pre-validate validate syntax etc. Works with most, but not all, Snowflake statements.') parser_render = subcommands.add_parser('render', description="Renders a script to the console, used to check and verify jinja output from scripts.") parser_render.add_argument('--config-folder', type = str, default = '.', help = 'The folder to look in for the schemachange-config.yml file (the default is the current working directory)', required = False) @@ -589,9 +613,9 @@ def main(): # First get the config values config_file_path = os.path.join(args.config_folder, _config_file_name) if args.subcommand == 'render': - config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, None, None, None, None, None, None, args.vars, None, None, args.verbose, None) + config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, None, None, None, None, None, None, args.vars, None, None, args.verbose, None, None) else: - config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, args.snowflake_account, args.snowflake_user, args.snowflake_role, args.snowflake_warehouse, args.snowflake_database, args.change_history_table, args.vars, args.create_change_history_table, args.autocommit, args.verbose, args.dry_run) + config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, args.snowflake_account, args.snowflake_user, args.snowflake_role, args.snowflake_warehouse, args.snowflake_database, args.change_history_table, args.vars, args.create_change_history_table, args.autocommit, args.verbose, args.dry_run, args.explain_first) # Then log some details print("Using root folder %s" % config['root-folder']) From e22a7f7c8a51f1115daf4ec583dcc8a999094049 Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Mon, 27 Sep 2021 14:34:46 +1000 Subject: [PATCH 6/9] fix --- schemachange/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 92f46ada..7c19f4cf 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -166,7 +166,7 @@ def deploy_command(config): continue if config['explain-first']: - explain_change_script(script, config['vars'], config['snowflake-database'], snowflake_session_parameters, config['autocommit'], config['verbose']) + explain_change_script(script, config['vars'], config['snowflake-database'], snowflake_session_parameters, config['verbose']) print("Applying change script %s" % script['script_name']) if not config['dry-run']: From 6f8faa2af3d7561f7d7ba0eccbc2fd3d7ef3e550 Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Thu, 4 Nov 2021 17:26:26 +1100 Subject: [PATCH 7/9] allow non-first statements in scripts to fail. --- schemachange/cli.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 7c19f4cf..87897276 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -18,6 +18,7 @@ from cryptography.hazmat.primitives.asymmetric import dsa from cryptography.hazmat.primitives import serialization import io +from traceback import print_exc # Set a few global variables here _schemachange_version = '3.2.0' @@ -566,9 +567,17 @@ def explain_change_script(script, vars, default_database, snowflake_session_para content_io = io.StringIO(content) statements = snowflake.connector.util_text.split_statements(content_io) - for s, _ in statements: + for n, (s, _) in enumerate(statements): + print(f'Explaining statement {n}...') explain = f"explain {s}" - execute_snowflake_query(default_database, explain, snowflake_session_parameters, False, verbose) + try: + execute_snowflake_query(default_database, explain, snowflake_session_parameters, False, verbose) + except: + if n == 0: + raise + else: + warnings.warn(f'Statement {n} failed, but allowing failures for n > 0.') + print_exc() def main(): From b3f3953c6f5fd5eafb5b9a95b49f473083f3f282 Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Thu, 4 Nov 2021 19:38:44 +1100 Subject: [PATCH 8/9] add minor test --- tests/test_main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_main.py b/tests/test_main.py index 68f84215..cc9483bf 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -20,6 +20,7 @@ 'autocommit': False, 'verbose': False, 'dry-run': False, + 'explain-first': False } @@ -50,6 +51,8 @@ {**DEFAULT_CONFIG, 'verbose': True}), (["schemachange", "deploy", "--dry-run"], {**DEFAULT_CONFIG, 'dry-run': True}), + (["schemachange", "deploy", "--explain-first"], + {**DEFAULT_CONFIG, 'explain-first': True}), ]) def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): sys.argv = args From 6746c5ef041260d9bc0384900e6eca0950cad623 Mon Sep 17 00:00:00 2001 From: Jarrad Whitaker Date: Thu, 4 Nov 2021 22:09:22 +1100 Subject: [PATCH 9/9] update for base refactor --- schemachange/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 87897276..9c5727d3 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -167,7 +167,7 @@ def deploy_command(config): continue if config['explain-first']: - explain_change_script(script, config['vars'], config['snowflake-database'], snowflake_session_parameters, config['verbose']) + explain_change_script(script, content, config['vars'], config['snowflake-database'], snowflake_session_parameters, config['verbose']) print("Applying change script %s" % script['script_name']) if not config['dry-run']: @@ -558,12 +558,11 @@ def apply_change_script(script, script_content, vars, default_database, change_h execute_snowflake_query(change_history_table['database_name'], query, snowflake_session_parameters, autocommit, verbose) -def explain_change_script(script, vars, default_database, snowflake_session_parameters, verbose): +def explain_change_script(script, content, vars, default_database, snowflake_session_parameters, verbose): ''' Run "explain " for every in a script of ; ; ... This will throw an error if the explain fails, which will catch many issues with the script without needing to directly execute it. ''' - content = get_script_contents_with_variable_replacement(script['script_full_path'], vars, verbose) content_io = io.StringIO(content) statements = snowflake.connector.util_text.split_statements(content_io)