From 71e8619ffe536ae6a07dd03b5b9e729545246582 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Mon, 11 Mar 2024 14:46:07 +0100 Subject: [PATCH] Snow 1181759 git setup (#880) * initial version of setup command * Quickcheck whether repository already exists * unit tests * integration tests * small fix; update help messages test * check setup output in integration tests * review: refactor git/commands::setup * review fixes * Update src/snowflake/cli/plugins/git/commands.py Co-authored-by: Tomasz Urbaszek * refactor setup command * update unit tests * update integration tests * multiline string --------- Co-authored-by: Tomasz Urbaszek --- src/snowflake/cli/api/sql_execution.py | 27 +++ src/snowflake/cli/plugins/git/commands.py | 95 ++++++++ src/snowflake/cli/plugins/git/manager.py | 16 ++ tests/__snapshots__/test_help_messages.ambr | 73 ++++++ tests/git/test_git_commands.py | 246 ++++++++++++++++++++ tests_integration/test_git.py | 33 +-- 6 files changed, 463 insertions(+), 27 deletions(-) diff --git a/src/snowflake/cli/api/sql_execution.py b/src/snowflake/cli/api/sql_execution.py index 3b2a04fcba..8512e26851 100644 --- a/src/snowflake/cli/api/sql_execution.py +++ b/src/snowflake/cli/api/sql_execution.py @@ -82,6 +82,33 @@ def use_role(self, new_role: str): if is_different_role: self._execute_query(f"use role {prev_role}") + def create_password_secret( + self, name: str, username: str, password: str + ) -> SnowflakeCursor: + query = dedent( + f""" + create secret {name} + type = password + username = '{username}' + password = '{password}' + """ + ) + return self._execute_query(query) + + def create_api_integration( + self, name: str, api_provider: str, allowed_prefix: str, secret: Optional[str] + ) -> SnowflakeCursor: + query = dedent( + f""" + create api integration {name} + api_provider = {api_provider} + api_allowed_prefixes = ('{allowed_prefix}') + allowed_authentication_secrets = ({secret if secret else ''}) + enabled = true + """ + ) + return self._execute_query(query) + def _execute_schema_query(self, query: str, name: Optional[str] = None, **kwargs): """ Check that a database and schema are provided before executing the query. Useful for operating on schema level objects. diff --git a/src/snowflake/cli/plugins/git/commands.py b/src/snowflake/cli/plugins/git/commands.py index 13be906546..8afc6808be 100644 --- a/src/snowflake/cli/plugins/git/commands.py +++ b/src/snowflake/cli/plugins/git/commands.py @@ -5,9 +5,13 @@ from click import ClickException from snowflake.cli.api.commands.flags import identifier_argument from snowflake.cli.api.commands.snow_typer import SnowTyper +from snowflake.cli.api.console.console import cli_console +from snowflake.cli.api.constants import ObjectType from snowflake.cli.api.output.types import CommandResult, QueryResult from snowflake.cli.api.utils.path_utils import is_stage_path from snowflake.cli.plugins.git.manager import GitManager +from snowflake.cli.plugins.object.manager import ObjectManager +from snowflake.connector import ProgrammingError app = SnowTyper( name="git", @@ -38,6 +42,97 @@ def _repo_path_argument_callback(path): ) +def _object_exists(object_type, identifier): + try: + ObjectManager().describe( + object_type=object_type.value.cli_name, + name=identifier, + ) + return True + except ProgrammingError: + return False + + +def _assure_repository_does_not_exist(repository_name: str) -> None: + if _object_exists(ObjectType.GIT_REPOSITORY, repository_name): + raise ClickException(f"Repository '{repository_name}' already exists") + + +def _validate_origin_url(url: str) -> None: + if not url.startswith("https://"): + raise ClickException("Url address should start with 'https'") + + +@app.command("setup", requires_connection=True) +def setup( + repository_name: str = RepoNameArgument, + **options, +) -> CommandResult: + """ + Sets up a git repository object. + + You will be prompted for: + + * url - address of repository to be used for git clone operation + + * secret - Snowflake secret containing authentication credentials. Not needed if origin repository does not require + authentication for RO operations (clone, fetch) + + * API integration - object allowing Snowflake to interact with git repository. + """ + _assure_repository_does_not_exist(repository_name) + manager = GitManager() + + url = typer.prompt("Origin url") + _validate_origin_url(url) + + secret = {} + secret_needed = typer.confirm("Use secret for authentication?") + if secret_needed: + secret_name = f"{repository_name}_secret" + secret_name = typer.prompt( + "Secret identifier (will be created if not exists)", default=secret_name + ) + secret = {"name": secret_name} + if _object_exists(ObjectType.SECRET, secret_name): + cli_console.step(f"Using existing secret '{secret_name}'") + else: + cli_console.step(f"Secret '{secret_name}' will be created") + secret["username"] = typer.prompt("username") + secret["password"] = typer.prompt("password/token", hide_input=True) + + api_integration = f"{repository_name}_api_integration" + api_integration = typer.prompt( + "API integration identifier (will be created if not exists)", + default=api_integration, + ) + + if "username" in secret: + manager.create_password_secret(**secret) + secret_name = secret["name"] + cli_console.step(f"Secret '{secret_name}' successfully created.") + + if not _object_exists(ObjectType.INTEGRATION, api_integration): + manager.create_api_integration( + name=api_integration, + api_provider="git_https_api", + allowed_prefix=url, + secret=secret.get("name"), + ) + cli_console.step(f"API integration '{api_integration}' successfully created.") + else: + cli_console.step(f"Using existing API integration '{api_integration}'.") + + return QueryResult( + manager.create( + repo_name=repository_name, + url=url, + api_integration=api_integration, + secret=secret.get("name"), + ) + ) + + @app.command( "list-branches", requires_connection=True, diff --git a/src/snowflake/cli/plugins/git/manager.py b/src/snowflake/cli/plugins/git/manager.py index 04e4816f4b..2c8e71cdc1 100644 --- a/src/snowflake/cli/plugins/git/manager.py +++ b/src/snowflake/cli/plugins/git/manager.py @@ -1,3 +1,5 @@ +from textwrap import dedent + from snowflake.cli.plugins.object.stage.manager import StageManager from snowflake.connector.cursor import SnowflakeCursor @@ -18,3 +20,17 @@ def show_files(self, repo_path: str) -> SnowflakeCursor: def fetch(self, repo_name: str) -> SnowflakeCursor: query = f"alter git repository {repo_name} fetch" return self._execute_query(query) + + def create( + self, repo_name: str, api_integration: str, url: str, secret: str + ) -> SnowflakeCursor: + query = dedent( + f""" + create git repository {repo_name} + api_integration = {api_integration} + origin = '{url}' + """ + ) + if secret is not None: + query += f"git_credentials = {secret}\n" + return self._execute_query(query) diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 4b5bcedb84..848ad50469 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -1127,6 +1127,78 @@ ╰──────────────────────────────────────────────────────────────────────────────╯ + ''' +# --- +# name: test_help_messages[git.setup] + ''' + + Usage: default git setup [OPTIONS] REPOSITORY_NAME + + Sets up a git repository object. + You will be prompted for: + * url - address of repository to be used for git clone operation + * secret - Snowflake secret containing authentication credentials. Not needed + if origin repository does not require authentication for RO operations (clone, + fetch) + * API integration - object allowing Snowflake to interact with git repository. + + ╭─ Arguments ──────────────────────────────────────────────────────────────────╮ + │ * repository_name TEXT Identifier of the git repository. For │ + │ example: my_repo │ + │ [default: None] │ + │ [required] │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + ╭─ Options ────────────────────────────────────────────────────────────────────╮ + │ --help -h Show this message and exit. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + ╭─ Connection configuration ───────────────────────────────────────────────────╮ + │ --connection,--environment -c TEXT Name of the connection, as defined │ + │ in your `config.toml`. Default: │ + │ `default`. │ + │ --account,--accountname TEXT Name assigned to your Snowflake │ + │ account. Overrides the value │ + │ specified for the connection. │ + │ --user,--username TEXT Username to connect to Snowflake. │ + │ Overrides the value specified for │ + │ the connection. │ + │ --password TEXT Snowflake password. Overrides the │ + │ value specified for the │ + │ connection. │ + │ --authenticator TEXT Snowflake authenticator. Overrides │ + │ the value specified for the │ + │ connection. │ + │ --private-key-path TEXT Snowflake private key path. │ + │ Overrides the value specified for │ + │ the connection. │ + │ --database,--dbname TEXT Database to use. Overrides the │ + │ value specified for the │ + │ connection. │ + │ --schema,--schemaname TEXT Database schema to use. Overrides │ + │ the value specified for the │ + │ connection. │ + │ --role,--rolename TEXT Role to use. Overrides the value │ + │ specified for the connection. │ + │ --warehouse TEXT Warehouse to use. Overrides the │ + │ value specified for the │ + │ connection. │ + │ --temporary-connection -x Uses connection defined with │ + │ command line parameters, instead │ + │ of one defined in config │ + │ --mfa-passcode TEXT Token to use for multi-factor │ + │ authentication (MFA) │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + ╭─ Global configuration ───────────────────────────────────────────────────────╮ + │ --format [TABLE|JSON] Specifies the output format. │ + │ [default: TABLE] │ + │ --verbose -v Displays log entries for log levels `info` │ + │ and higher. │ + │ --debug Displays log entries for log levels `debug` │ + │ and higher; debug logs contains additional │ + │ information. │ + │ --silent Turns off intermediate output to console. │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + ''' # --- # name: test_help_messages[git] @@ -1146,6 +1218,7 @@ │ list-branches List all branches in the repository. │ │ list-files List files from given state of git repository. │ │ list-tags List all tags in the repository. │ + │ setup Sets up a git repository object. │ ╰──────────────────────────────────────────────────────────────────────────────╯ diff --git a/tests/git/test_git_commands.py b/tests/git/test_git_commands.py index fab949042b..492cf48e9c 100644 --- a/tests/git/test_git_commands.py +++ b/tests/git/test_git_commands.py @@ -1,7 +1,11 @@ from pathlib import Path +from textwrap import dedent from unittest import mock import pytest +from snowflake.connector import ProgrammingError + +EXAMPLE_URL = "https://github.com/an-example-repo.git" @pytest.mark.skip(reason="Command is hidden") @@ -99,6 +103,248 @@ def test_copy_not_a_stage_error(runner): _assert_invalid_repo_path_error_message(result.output) +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.snowpark.commands.ObjectManager.describe") +def test_setup_already_exists_error(mock_om_describe, mock_connector, runner, mock_ctx): + mock_om_describe.return_value = {"object_details": "something"} + ctx = mock_ctx() + mock_connector.return_value = ctx + + result = runner.invoke(["git", "setup", "repo_name"]) + assert result.exit_code == 1, result.output + assert "Error" in result.output + assert "Repository 'repo_name' already exists" in result.output + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.snowpark.commands.ObjectManager.describe") +def test_setup_invalid_url_error(mock_om_describe, mock_connector, runner, mock_ctx): + mock_om_describe.side_effect = ProgrammingError("does not exist or not authorized") + ctx = mock_ctx() + mock_connector.return_value = ctx + communication = "http://invalid_url.git\ns" + result = runner.invoke(["git", "setup", "repo_name"], input=communication) + + assert result.exit_code == 1, result.output + assert "Error" in result.output + assert "Url address should start with 'https'" in result.output + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.snowpark.commands.ObjectManager.describe") +def test_setup_no_secret_existing_api( + mock_om_describe, mock_connector, runner, mock_ctx +): + mock_om_describe.side_effect = [ + ProgrammingError("does not exist or not authorized"), + None, + ] + mock_om_describe.return_value = [None, {"object_details": "something"}] + ctx = mock_ctx() + mock_connector.return_value = ctx + + communication = "\n".join([EXAMPLE_URL, "n", "existing_api_integration", ""]) + result = runner.invoke(["git", "setup", "repo_name"], input=communication) + + assert result.exit_code == 0, result.output + assert result.output.startswith( + "\n".join( + [ + "Origin url: https://github.com/an-example-repo.git", + "Use secret for authentication? [y/N]: n", + "API integration identifier (will be created if not exists) [repo_name_api_integration]: existing_api_integration", + "Using existing API integration 'existing_api_integration'.", + ] + ) + ) + assert ctx.get_query() == dedent( + """ + create git repository repo_name + api_integration = existing_api_integration + origin = 'https://github.com/an-example-repo.git' + """ + ) + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.snowpark.commands.ObjectManager.describe") +def test_setup_no_secret_create_api(mock_om_describe, mock_connector, runner, mock_ctx): + mock_om_describe.side_effect = ProgrammingError("does not exist or not authorized") + ctx = mock_ctx() + mock_connector.return_value = ctx + + communication = "\n".join([EXAMPLE_URL, "n", "", ""]) + result = runner.invoke(["git", "setup", "repo_name"], input=communication) + + assert result.exit_code == 0, result.output + assert result.output.startswith( + "\n".join( + [ + "Origin url: https://github.com/an-example-repo.git", + "Use secret for authentication? [y/N]: n", + "API integration identifier (will be created if not exists) [repo_name_api_integration]: ", + "API integration 'repo_name_api_integration' successfully created.", + ] + ) + ) + assert ctx.get_query() == dedent( + """ + create api integration repo_name_api_integration + api_provider = git_https_api + api_allowed_prefixes = ('https://github.com/an-example-repo.git') + allowed_authentication_secrets = () + enabled = true + + + create git repository repo_name + api_integration = repo_name_api_integration + origin = 'https://github.com/an-example-repo.git' + """ + ) + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.snowpark.commands.ObjectManager.describe") +def test_setup_existing_secret_existing_api( + mock_om_describe, mock_connector, runner, mock_ctx +): + mock_om_describe.side_effect = [ + ProgrammingError("does not exist or not authorized"), + None, + None, + ] + mock_om_describe.return_value = [None, "integration_details", "secret_details"] + ctx = mock_ctx() + mock_connector.return_value = ctx + + communication = "\n".join( + [EXAMPLE_URL, "y", "existing_secret", "existing_api_integration", ""] + ) + result = runner.invoke(["git", "setup", "repo_name"], input=communication) + + assert result.exit_code == 0, result.output + assert result.output.startswith( + "\n".join( + [ + "Origin url: https://github.com/an-example-repo.git", + "Use secret for authentication? [y/N]: y", + "Secret identifier (will be created if not exists) [repo_name_secret]: existing_secret", + "Using existing secret 'existing_secret'", + "API integration identifier (will be created if not exists) [repo_name_api_integration]: existing_api_integration", + "Using existing API integration 'existing_api_integration'.", + ] + ) + ) + assert ctx.get_query() == dedent( + """ + create git repository repo_name + api_integration = existing_api_integration + origin = 'https://github.com/an-example-repo.git' + git_credentials = existing_secret + """ + ) + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.snowpark.commands.ObjectManager.describe") +def test_setup_existing_secret_create_api( + mock_om_describe, mock_connector, runner, mock_ctx +): + mock_om_describe.side_effect = [ + ProgrammingError("does not exist or not authorized"), + None, + ProgrammingError("does not exist or not authorized"), + ] + mock_om_describe.return_value = [None, "secret_details", None] + ctx = mock_ctx() + mock_connector.return_value = ctx + + communication = "\n".join([EXAMPLE_URL, "y", "existing_secret", "", ""]) + result = runner.invoke(["git", "setup", "repo_name"], input=communication) + + assert result.exit_code == 0, result.output + assert result.output.startswith( + "\n".join( + [ + "Origin url: https://github.com/an-example-repo.git", + "Use secret for authentication? [y/N]: y", + "Secret identifier (will be created if not exists) [repo_name_secret]: existing_secret", + "Using existing secret 'existing_secret'", + "API integration identifier (will be created if not exists) [repo_name_api_integration]: ", + "API integration 'repo_name_api_integration' successfully created.", + ] + ) + ) + assert ctx.get_query() == dedent( + """ + create api integration repo_name_api_integration + api_provider = git_https_api + api_allowed_prefixes = ('https://github.com/an-example-repo.git') + allowed_authentication_secrets = (existing_secret) + enabled = true + + + create git repository repo_name + api_integration = repo_name_api_integration + origin = 'https://github.com/an-example-repo.git' + git_credentials = existing_secret + """ + ) + + +@mock.patch("snowflake.connector.connect") +@mock.patch("snowflake.cli.plugins.snowpark.commands.ObjectManager.describe") +def test_setup_create_secret_create_api( + mock_om_describe, mock_connector, runner, mock_ctx +): + mock_om_describe.side_effect = ProgrammingError("does not exist or not authorized") + ctx = mock_ctx() + mock_connector.return_value = ctx + + communication = "\n".join( + [EXAMPLE_URL, "y", "", "john_doe", "admin123", "new_integration", ""] + ) + result = runner.invoke(["git", "setup", "repo_name"], input=communication) + + assert result.exit_code == 0, result.output + assert result.output.startswith( + "\n".join( + [ + "Origin url: https://github.com/an-example-repo.git", + "Use secret for authentication? [y/N]: y", + "Secret identifier (will be created if not exists) [repo_name_secret]: ", + "Secret 'repo_name_secret' will be created", + "username: john_doe", + "password/token: ", + "API integration identifier (will be created if not exists) [repo_name_api_integration]: new_integration", + "Secret 'repo_name_secret' successfully created.", + "API integration 'new_integration' successfully created.", + ] + ) + ) + assert ctx.get_query() == dedent( + """ + create secret repo_name_secret + type = password + username = 'john_doe' + password = 'admin123' + + + create api integration new_integration + api_provider = git_https_api + api_allowed_prefixes = ('https://github.com/an-example-repo.git') + allowed_authentication_secrets = (repo_name_secret) + enabled = true + + + create git repository repo_name + api_integration = new_integration + origin = 'https://github.com/an-example-repo.git' + git_credentials = repo_name_secret + """ + ) + + def _assert_invalid_repo_path_error_message(output): assert "Error" in output assert ( diff --git a/tests_integration/test_git.py b/tests_integration/test_git.py index 5ae9e1a96a..5846c837e1 100644 --- a/tests_integration/test_git.py +++ b/tests_integration/test_git.py @@ -9,36 +9,15 @@ @pytest.fixture def sf_git_repository(runner, test_database): repo_name = "SNOWCLI_TESTING_REPO" - integration_name = "SNOW_GIT_TESTING_API_INTEGRATION" - - if not _integration_exists(runner, integration_name=integration_name): - result = runner.invoke_with_connection( - [ - "sql", - "-q", - f""" - CREATE API INTEGRATION {integration_name} - API_PROVIDER = git_https_api - API_ALLOWED_PREFIXES = ('https://github.com/snowflakedb/') - ALLOWED_AUTHENTICATION_SECRETS = () - ENABLED = true - """, - ] - ) - assert result.exit_code == 0 - + integration_name = "SNOWCLI_TESTING_REPO_API_INTEGRATION" + communication = "\n".join( + ["https://github.com/snowflakedb/snowflake-cli.git", "n", integration_name, ""] + ) result = runner.invoke_with_connection( - [ - "sql", - "-q", - f""" - CREATE GIT REPOSITORY {repo_name} - API_INTEGRATION = {integration_name} - ORIGIN = 'https://github.com/snowflakedb/snowflake-cli.git' - """, - ] + ["git", "setup", repo_name], input=communication ) assert result.exit_code == 0 + assert f"Git Repository {repo_name} was successfully created." in result.output return repo_name