Add "--explain-first" to run explain on statements

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Python tests
+
+on: [push]
+
+jobs:
+  build:
+
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+        github-runner: ['ubuntu-latest', 'windows-latest']
+
+    runs-on: ${{ matrix.github-runner }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install -e .[dev]
+      - name: Test with pytest
+        run: |
+          pytest

schemachange/cli.py

@@ -17,6 +17,8 @@
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.primitives.asymmetric import dsa
 from cryptography.hazmat.primitives import serialization
+import io
+from traceback import print_exc
 
 # Set a few global variables here
 _schemachange_version = '3.2.0'
@@ -164,6 +166,9 @@ def deploy_command(config):
         scripts_skipped += 1
         continue
 
+    if config['explain-first']:
+      explain_change_script(script, content, config['vars'], config['snowflake-database'], snowflake_session_parameters, config['verbose'])
+
     print("Applying change script %s" % script['script_name'])
     if not config['dry-run']:
       apply_change_script(script, content, config['vars'], config['snowflake-database'], change_history_table, snowflake_session_parameters, config['autocommit'], config['verbose'])
@@ -200,7 +205,7 @@ def get_alphanum_key(key):
 def sorted_alphanumeric(data):
   return sorted(data, key=get_alphanum_key)
 
-def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, change_history_table_override, vars, create_change_history_table, autocommit, verbose, dry_run):
+def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, change_history_table_override, vars, create_change_history_table, autocommit, verbose, dry_run, explain_first):
   config = dict()
 
   # First read in the yaml config file, if present
@@ -294,6 +299,11 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf
     if "schemachange" in config['vars']:
       raise ValueError("The variable schemachange has been reserved for use by schemachange, please use a different name")
 
+  if explain_first:
+    config['explain-first'] = explain_first
+  if 'explain-first' not in config:
+    config['explain-first'] = False
+
   return config
 
 def get_all_scripts_recursively(root_directory, verbose):
@@ -548,6 +558,27 @@ def apply_change_script(script, script_content, vars, default_database, change_h
   execute_snowflake_query(change_history_table['database_name'], query, snowflake_session_parameters, autocommit, verbose)
 
 
+def explain_change_script(script, content, vars, default_database, snowflake_session_parameters, verbose):
+  '''
+  Run "explain <statement>" for every <statement> in a script of <statement>; <statement>; ...
+  This will throw an error if the explain fails, which will catch many issues with the script without needing to directly execute it.
+  '''
+
+  content_io = io.StringIO(content)
+  statements = snowflake.connector.util_text.split_statements(content_io)
+  for n, (s, _) in enumerate(statements):
+      print(f'Explaining statement {n}...')
+      explain = f"explain {s}"
+      try:
+        execute_snowflake_query(default_database, explain, snowflake_session_parameters, False, verbose)
+      except:
+        if n == 0:
+          raise
+        else:
+          warnings.warn(f'Statement {n} failed, but allowing failures for n > 0.')
+          print_exc()
+
+
 def main():
   parser = argparse.ArgumentParser(prog = 'schemachange', description = 'Apply schema changes to a Snowflake account. Full readme at https://github.com/Snowflake-Labs/schemachange', formatter_class = argparse.RawTextHelpFormatter)
   subcommands = parser.add_subparsers(dest='subcommand')
@@ -567,6 +598,7 @@ def main():
   parser_deploy.add_argument('-ac', '--autocommit', action='store_true', help = 'Enable autocommit feature for DML commands (the default is False)', required = False)
   parser_deploy.add_argument('-v','--verbose', action='store_true', help = 'Display verbose debugging details during execution (the default is False)', required = False)
   parser_deploy.add_argument('--dry-run', action='store_true', help = 'Run schemachange in dry run mode (the default is False)', required = False)
+  parser_deploy.add_argument('--explain-first', action='store_true', help = 'Run explain <statement> before each query is run properly, which can be used to pre-validate validate syntax etc. Works with most, but not all, Snowflake statements.')
 
   parser_render = subcommands.add_parser('render', description="Renders a script to the console, used to check and verify jinja output from scripts.")
   parser_render.add_argument('--config-folder', type = str, default = '.', help = 'The folder to look in for the schemachange-config.yml file (the default is the current working directory)', required = False)
@@ -589,9 +621,9 @@ def main():
   # First get the config values
   config_file_path = os.path.join(args.config_folder, _config_file_name)
   if args.subcommand == 'render':
-    config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, None, None, None, None, None, None, args.vars, None, None, args.verbose, None)
+    config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, None, None, None, None, None, None, args.vars, None, None, args.verbose, None, None)
   else:
-    config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, args.snowflake_account, args.snowflake_user, args.snowflake_role, args.snowflake_warehouse, args.snowflake_database, args.change_history_table, args.vars, args.create_change_history_table, args.autocommit, args.verbose, args.dry_run)
+    config = get_schemachange_config(config_file_path, args.root_folder, args.modules_folder, args.snowflake_account, args.snowflake_user, args.snowflake_role, args.snowflake_warehouse, args.snowflake_database, args.change_history_table, args.vars, args.create_change_history_table, args.autocommit, args.verbose, args.dry_run, args.explain_first)
 
   # Then log some details
   print("Using root folder %s" % config['root-folder'])

tests/test_main.py

@@ -2,71 +2,130 @@
 import sys
 import pytest
 import unittest.mock as mock
-import schemachange.cli 
+import schemachange.cli
+import tempfile
+from textwrap import dedent
 
-
-@pytest.mark.parametrize("args, expected", [
-    (["schemachange"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "--config-folder", "test"], ('test', None, None, None, None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "-f", '.'], ('.', '.', None, None, None, None, None, None, None, None, False, False, False, False)),     
-    (["schemachange", "--modules-folder", "modules-folder"], ('.', None, "modules-folder", None, None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "--snowflake-account", "account"], ('.', None, None, "account", None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "--snowflake-user", "user"], ('.', None, None, None, "user", None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "--snowflake-role", "role"], ('.', None, None, None, None, "role", None, None, None, None, False, False, False, False)),
-    (["schemachange", "--snowflake-warehouse", "warehouse"], ('.', None, None, None, None, None, "warehouse", None, None, None, False, False, False, False)),
-    (["schemachange", "--snowflake-database", "database"], ('.', None, None, None, None, None, None, "database", None, None, False, False, False, False)),    
-    (["schemachange", "--change-history-table", "db.schema.table"], ('.', None, None, None, None, None, None, None, "db.schema.table", None, False, False, False, False)),    
-    (["schemachange", "--vars", '{"var1": "val"}'], ('.', None, None, None, None, None, None, None, None, {'var1' : 'val'}, False, False, False, False)),
-    (["schemachange", "--create-change-history-table"], ('.', None, None, None, None, None, None, None, None, None, True, False, False, False)),
-    (["schemachange", "--autocommit"], ('.', None, None, None, None, None, None, None, None, None, False, True, False, False)),
-    (["schemachange", "--verbose"], ('.', None, None, None, None, None, None, None, None, None, False, False, True, False)),
-    (["schemachange", "--dry-run"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, True))       
-])
-def test_main_no_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected):
-    sys.argv = args
-
-    with mock.patch("schemachange.cli.schemachange") as mock_schemachange:
-        schemachange.cli.main()
-        mock_schemachange.assert_called_once_with(*expected)
+DEFAULT_CONFIG = {
+    'root-folder': os.path.abspath('.'),
+    'modules-folder': None,
+    'snowflake-account': None,
+    'snowflake-user': None,
+    'snowflake-role':None,
+    'snowflake-warehouse': None,
+    'snowflake-database': None,
+    'change-history-table': None,
+    'vars': {},
+    'create-change-history-table': False,
+    'autocommit': False,
+    'verbose': False,
+    'dry-run': False,
+    'explain-first': False
+}
 
 
 @pytest.mark.parametrize("args, expected", [
-    (["schemachange", "deploy"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "deploy", "--config-folder", "test"], ('test', None, None, None, None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "deploy", "-f", '.'], ('.', '.', None, None, None, None, None, None, None, None, False, False, False, False)),     
-    (["schemachange", "deploy", "--modules-folder", "modules-folder"], ('.', None, "modules-folder", None, None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "deploy", "--snowflake-account", "account"], ('.', None, None, "account", None, None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "deploy", "--snowflake-user", "user"], ('.', None, None, None, "user", None, None, None, None, None, False, False, False, False)),
-    (["schemachange", "deploy", "--snowflake-role", "role"], ('.', None, None, None, None, "role", None, None, None, None, False, False, False, False)),
-    (["schemachange", "deploy", "--snowflake-warehouse", "warehouse"], ('.', None, None, None, None, None, "warehouse", None, None, None, False, False, False, False)),
-    (["schemachange", "deploy", "--snowflake-database", "database"], ('.', None, None, None, None, None, None, "database", None, None, False, False, False, False)),    
-    (["schemachange", "deploy", "--change-history-table", "db.schema.table"], ('.', None, None, None, None, None, None, None, "db.schema.table", None, False, False, False, False)),    
-    (["schemachange", "deploy", "--vars", '{"var1": "val"}'], ('.', None, None, None, None, None, None, None, None, {'var1' : 'val'}, False, False, False, False)),
-    (["schemachange", "deploy", "--create-change-history-table"], ('.', None, None, None, None, None, None, None, None, None, True, False, False, False)),
-    (["schemachange", "deploy", "--autocommit"], ('.', None, None, None, None, None, None, None, None, None, False, True, False, False)),
-    (["schemachange", "deploy", "--verbose"], ('.', None, None, None, None, None, None, None, None, None, False, False, True, False)),
-    (["schemachange", "deploy", "--dry-run"], ('.', None, None, None, None, None, None, None, None, None, False, False, False, True))
-
+    (["schemachange"], DEFAULT_CONFIG),
+    (["schemachange", "deploy"], DEFAULT_CONFIG),
+    (["schemachange", "deploy", "-f", '.'],
+        {**DEFAULT_CONFIG, 'root-folder':os.path.abspath('.')}),
+    (["schemachange", "deploy", "--snowflake-account", "account"],
+        {**DEFAULT_CONFIG, 'snowflake-account': 'account'}),
+    (["schemachange", "deploy", "--snowflake-user", "user"],
+        {**DEFAULT_CONFIG, 'snowflake-user': 'user'}),
+    (["schemachange", "deploy", "--snowflake-role", "role"],
+        {**DEFAULT_CONFIG, 'snowflake-role': 'role'}),
+    (["schemachange", "deploy", "--snowflake-warehouse", "warehouse"],
+        {**DEFAULT_CONFIG, 'snowflake-warehouse': 'warehouse'}),
+    (["schemachange", "deploy", "--snowflake-database", "database"],
+        {**DEFAULT_CONFIG, 'snowflake-database': 'database'}),
+    (["schemachange", "deploy", "--change-history-table", "db.schema.table"],
+        {**DEFAULT_CONFIG, 'change-history-table': 'db.schema.table'}),
+    (["schemachange", "deploy", "--vars", '{"var1": "val"}'],
+        {**DEFAULT_CONFIG, 'vars': {'var1' : 'val'},}),
+    (["schemachange", "deploy", "--create-change-history-table"],
+        {**DEFAULT_CONFIG, 'create-change-history-table': True}),
+    (["schemachange", "deploy", "--autocommit"],
+        {**DEFAULT_CONFIG, 'autocommit': True}),
+    (["schemachange", "deploy", "--verbose"],
+        {**DEFAULT_CONFIG, 'verbose': True}),
+    (["schemachange", "deploy", "--dry-run"],
+        {**DEFAULT_CONFIG, 'dry-run': True}),
+    (["schemachange", "deploy", "--explain-first"],
+        {**DEFAULT_CONFIG, 'explain-first': True}),
 ])
 def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected):
     sys.argv = args
-    
-    with mock.patch("schemachange.cli.schemachange") as mock_schemachange:
+
+    with mock.patch("schemachange.cli.deploy_command") as mock_deploy_command:
         schemachange.cli.main()
-        mock_schemachange.assert_called_once_with(*expected)
+        mock_deploy_command.assert_called_once()
+        [config,], _call_kwargs = mock_deploy_command.call_args
+        assert config == expected
 
 
 @pytest.mark.parametrize("args, expected", [
-    (["schemachange", "render", "script.sql"], ('.', None, None, None, False, "script.sql")),
-    (["schemachange", "render", "--config-folder", "test", "script.sql"], ("test", None, None, None, False, "script.sql")),
-    (["schemachange", "render", "--root-folder", '.', "script.sql"], ('.', ".", None, None, False, "script.sql")),
-    (["schemachange", "render", "--modules-folder", "modules-folder", "script.sql"], ('.', None, "modules-folder", None, False, "script.sql")),
-    (["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"], ('.', None, None, {'var1' : 'val'}, False, "script.sql")),
-    (["schemachange", "render", "--verbose", "script.sql"], ('.', None, None, None, True, "script.sql")),      
+    (["schemachange", "render", "script.sql"],
+        ({**DEFAULT_CONFIG}, "script.sql")),
+    (["schemachange", "render", "--root-folder", '.', "script.sql"],
+        ({**DEFAULT_CONFIG, 'root-folder': os.path.abspath('.')}, "script.sql")),
+    (["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"],
+        ({**DEFAULT_CONFIG, 'vars': {"var1": "val"}}, "script.sql")),
+    (["schemachange", "render", "--verbose", "script.sql"],
+        ({**DEFAULT_CONFIG, 'verbose': True}, "script.sql")),
 ])
 def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected):
     sys.argv = args
-    
+
     with mock.patch("schemachange.cli.render_command") as mock_render_command:
         schemachange.cli.main()
-        mock_render_command.assert_called_once_with(*expected)
+        mock_render_command.assert_called_once()
+        call_args, _call_kwargs = mock_render_command.call_args
+        assert call_args == expected
+
+
+@pytest.mark.parametrize("args, to_mock, expected_args", [
+    (["schemachange", "deploy", "--config-folder", "DUMMY"],
+        "schemachange.cli.deploy_command",
+        ({**DEFAULT_CONFIG, 'snowflake-account': 'account'},)),
+    (["schemachange", "render", "script.sql", "--config-folder", "DUMMY"],
+        "schemachange.cli.render_command",
+        ({**DEFAULT_CONFIG, 'snowflake-account': 'account'}, "script.sql"))
+])
+def test_main_deploy_config_folder(args, to_mock, expected_args):
+    with tempfile.TemporaryDirectory() as d:
+        with open(os.path.join(d, 'schemachange-config.yml'), 'wt') as f:
+            f.write(dedent('''
+                snowflake-account: account
+            '''))
+
+        args[args.index("DUMMY")] = d
+        sys.argv = args
+
+        with mock.patch(to_mock) as mock_command:
+            schemachange.cli.main()
+            mock_command.assert_called_once()
+            call_args, _call_kwargs = mock_command.call_args
+            assert call_args == expected_args
+
+
+@pytest.mark.parametrize("args, to_mock, expected_args", [
+    (["schemachange", "deploy", "--modules-folder", "DUMMY"],
+        "schemachange.cli.deploy_command",
+        ({**DEFAULT_CONFIG, 'modules-folder': 'DUMMY'},)),
+    (["schemachange", "render", "script.sql", "--modules-folder", "DUMMY"],
+        "schemachange.cli.render_command",
+        ({**DEFAULT_CONFIG, 'modules-folder': 'DUMMY'}, "script.sql"))
+])
+def test_main_deploy_modules_folder(args, to_mock, expected_args):
+    with tempfile.TemporaryDirectory() as d:
+
+        args[args.index("DUMMY")] = d
+        expected_args[0]['modules-folder'] = d
+        sys.argv = args
+
+        with mock.patch(to_mock) as mock_command:
+            schemachange.cli.main()
+            mock_command.assert_called_once()
+            call_args, _call_kwargs = mock_command.call_args
+            assert call_args == expected_args