diff --git a/src/snowflake/cli/_plugins/nativeapp/manager.py b/src/snowflake/cli/_plugins/nativeapp/manager.py index 95284e1e33..0965220b77 100644 --- a/src/snowflake/cli/_plugins/nativeapp/manager.py +++ b/src/snowflake/cli/_plugins/nativeapp/manager.py @@ -15,7 +15,6 @@ from __future__ import annotations import json -import os import time from abc import ABC, abstractmethod from contextlib import contextmanager @@ -31,7 +30,6 @@ from snowflake.cli._plugins.nativeapp.artifacts import ( BundleMap, build_bundle, - resolve_without_follow, ) from snowflake.cli._plugins.nativeapp.codegen.compiler import ( NativeAppCompiler, @@ -49,17 +47,10 @@ from snowflake.cli._plugins.nativeapp.project_model import ( NativeAppProjectModel, ) -from snowflake.cli._plugins.nativeapp.utils import verify_exists, verify_no_directories from snowflake.cli._plugins.stage.diff import ( DiffResult, - StagePath, - compute_stage_diff, - preserve_from_diff, - sync_local_diff_with_stage, - to_stage_path, ) from snowflake.cli._plugins.stage.manager import StageManager -from snowflake.cli._plugins.stage.utils import print_diff_to_console from snowflake.cli.api.cli_global_context import get_cli_context from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.entities.application_package_entity import ( @@ -67,6 +58,7 @@ ) from snowflake.cli.api.entities.utils import ( generic_sql_error_handler, + sync_deploy_root_with_stage, ) from snowflake.cli.api.errno import ( DOES_NOT_EXIST_OR_NOT_AUTHORIZED, @@ -90,23 +82,23 @@ ApplicationOwnedObject = TypedDict("ApplicationOwnedObject", {"name": str, "type": str}) -def _get_stage_paths_to_sync( - local_paths_to_sync: List[Path], deploy_root: Path -) -> List[StagePath]: - """ - Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root. - """ +# def _get_stage_paths_to_sync( +# local_paths_to_sync: List[Path], deploy_root: Path +# ) -> List[StagePath]: +# """ +# Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root. +# """ - stage_paths = [] - for path in local_paths_to_sync: - if path.is_dir(): - for current_dir, _dirs, files in os.walk(path): - for file in files: - deploy_path = Path(current_dir, file).relative_to(deploy_root) - stage_paths.append(to_stage_path(deploy_path)) - else: - stage_paths.append(to_stage_path(path.relative_to(deploy_root))) - return stage_paths +# stage_paths = [] +# for path in local_paths_to_sync: +# if path.is_dir(): +# for current_dir, _dirs, files in os.walk(path): +# for file in files: +# deploy_path = Path(current_dir, file).relative_to(deploy_root) +# stage_paths.append(to_stage_path(deploy_path)) +# else: +# stage_paths.append(to_stage_path(path.relative_to(deploy_root))) +# return stage_paths class NativeAppCommandProcessor(ABC): @@ -288,110 +280,20 @@ def sync_deploy_root_with_stage( local_paths_to_sync: List[Path] | None = None, print_diff: bool = True, ) -> DiffResult: - """ - Ensures that the files on our remote stage match the artifacts we have in - the local filesystem. - - Args: - bundle_map (BundleMap): The artifact mapping computed by the `build_bundle` function. - role (str): The name of the role to use for queries and commands. - prune (bool): Whether to prune artifacts from the stage that don't exist locally. - recursive (bool): Whether to traverse directories recursively. - stage_fqn (str): The name of the stage to diff against and upload to. - local_paths_to_sync (List[Path], optional): List of local paths to sync. Defaults to None to sync all - local paths. Note that providing an empty list here is equivalent to None. - print_diff (bool): Whether to print the diff between the local files and the remote stage. Defaults to True - - Returns: - A `DiffResult` instance describing the changes that were performed. - """ - - # Does a stage already exist within the application package, or we need to create one? - # Using "if not exists" should take care of either case. - cc.step( - f"Checking if stage {stage_fqn} exists, or creating a new one if none exists." + return sync_deploy_root_with_stage( + sql_executor=self, + console=cc, + deploy_root=self.deploy_root, + package_name=self.package_name, + stage_schema=self.stage_schema, + bundle_map=bundle_map, + role=role, + prune=prune, + recursive=recursive, + stage_fqn=stage_fqn, + local_paths_to_sync=local_paths_to_sync, + print_diff=print_diff, ) - with self.use_role(role): - self._execute_query( - f"create schema if not exists {self.package_name}.{self.stage_schema}" - ) - self._execute_query( - f""" - create stage if not exists {stage_fqn} - encryption = (TYPE = 'SNOWFLAKE_SSE') - DIRECTORY = (ENABLE = TRUE)""" - ) - - # Perform a diff operation and display results to the user for informational purposes - if print_diff: - cc.step( - "Performing a diff between the Snowflake stage and your local deploy_root ('%s') directory." - % self.deploy_root.resolve() - ) - diff: DiffResult = compute_stage_diff(self.deploy_root, stage_fqn) - - if local_paths_to_sync: - # Deploying specific files/directories - resolved_paths_to_sync = [ - resolve_without_follow(p) for p in local_paths_to_sync - ] - if not recursive: - verify_no_directories(resolved_paths_to_sync) - - deploy_paths_to_sync = [] - for resolved_path in resolved_paths_to_sync: - verify_exists(resolved_path) - deploy_paths = bundle_map.to_deploy_paths(resolved_path) - if not deploy_paths: - if resolved_path.is_dir() and recursive: - # No direct artifact mapping found for this path. Check to see - # if there are subpaths of this directory that are matches. We - # loop over sources because it's likely a much smaller list - # than the project directory. - for src in bundle_map.all_sources(absolute=True): - if resolved_path in src.parents: - # There is a source that contains this path, get its dest path(s) - deploy_paths.extend(bundle_map.to_deploy_paths(src)) - - if not deploy_paths: - raise ClickException(f"No artifact found for {resolved_path}") - deploy_paths_to_sync.extend(deploy_paths) - - stage_paths_to_sync = _get_stage_paths_to_sync( - deploy_paths_to_sync, resolve_without_follow(self.deploy_root) - ) - diff = preserve_from_diff(diff, stage_paths_to_sync) - else: - # Full deploy - if not recursive: - verify_no_directories(self.deploy_root.resolve().iterdir()) - - if not prune: - files_not_removed = [str(path) for path in diff.only_on_stage] - diff.only_on_stage = [] - - if len(files_not_removed) > 0: - files_not_removed_str = "\n".join(files_not_removed) - cc.warning( - f"The following files exist only on the stage:\n{files_not_removed_str}\n\nUse the --prune flag to delete them from the stage." - ) - - if print_diff: - print_diff_to_console(diff, bundle_map) - - # Upload diff-ed files to application package stage - if diff.has_changes(): - cc.step( - "Updating the Snowflake stage from your local %s directory." - % self.deploy_root.resolve(), - ) - sync_local_diff_with_stage( - role=role, - deploy_root_path=self.deploy_root, - diff_result=diff, - stage_fqn=stage_fqn, - ) - return diff def get_existing_app_info(self) -> Optional[dict]: """ diff --git a/src/snowflake/cli/api/entities/utils.py b/src/snowflake/cli/api/entities/utils.py index b559e3ab9b..747ebac5b0 100644 --- a/src/snowflake/cli/api/entities/utils.py +++ b/src/snowflake/cli/api/entities/utils.py @@ -1,13 +1,32 @@ +import os +from pathlib import Path from textwrap import dedent -from typing import NoReturn, Optional +from typing import List, NoReturn, Optional +from click import ClickException +from snowflake.cli._plugins.nativeapp.artifacts import ( + BundleMap, + resolve_without_follow, +) from snowflake.cli._plugins.nativeapp.constants import OWNER_COL from snowflake.cli._plugins.nativeapp.exceptions import UnexpectedOwnerError +from snowflake.cli._plugins.nativeapp.utils import verify_exists, verify_no_directories +from snowflake.cli._plugins.stage.diff import ( + DiffResult, + StagePath, + compute_stage_diff, + preserve_from_diff, + sync_local_diff_with_stage, + to_stage_path, +) +from snowflake.cli._plugins.stage.utils import print_diff_to_console +from snowflake.cli.api.console.abc import AbstractConsole from snowflake.cli.api.errno import ( DOES_NOT_EXIST_OR_CANNOT_BE_PERFORMED, NO_WAREHOUSE_SELECTED_IN_SESSION, ) from snowflake.cli.api.project.util import unquote_identifier +from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector import ProgrammingError @@ -57,3 +76,142 @@ def ensure_correct_owner(row: dict, role: str, obj_name: str) -> None: ].upper() # Because unquote_identifier() always returns uppercase str if actual_owner != unquote_identifier(role): raise UnexpectedOwnerError(obj_name, role, actual_owner) + + +def _get_stage_paths_to_sync( + local_paths_to_sync: List[Path], deploy_root: Path +) -> List[StagePath]: + """ + Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root. + """ + + stage_paths = [] + for path in local_paths_to_sync: + if path.is_dir(): + for current_dir, _dirs, files in os.walk(path): + for file in files: + deploy_path = Path(current_dir, file).relative_to(deploy_root) + stage_paths.append(to_stage_path(deploy_path)) + else: + stage_paths.append(to_stage_path(path.relative_to(deploy_root))) + return stage_paths + + +def sync_deploy_root_with_stage( + sql_executor: SqlExecutionMixin, + console: AbstractConsole, + deploy_root: Path, + package_name: str, + stage_schema: str, + bundle_map: BundleMap, + role: str, + prune: bool, + recursive: bool, + stage_fqn: str, + local_paths_to_sync: List[Path] | None = None, + print_diff: bool = True, +) -> DiffResult: + """ + Ensures that the files on our remote stage match the artifacts we have in + the local filesystem. + + Args: + bundle_map (BundleMap): The artifact mapping computed by the `build_bundle` function. + role (str): The name of the role to use for queries and commands. + prune (bool): Whether to prune artifacts from the stage that don't exist locally. + recursive (bool): Whether to traverse directories recursively. + stage_fqn (str): The name of the stage to diff against and upload to. + local_paths_to_sync (List[Path], optional): List of local paths to sync. Defaults to None to sync all + local paths. Note that providing an empty list here is equivalent to None. + print_diff (bool): Whether to print the diff between the local files and the remote stage. Defaults to True + + Returns: + A `DiffResult` instance describing the changes that were performed. + """ + + # Does a stage already exist within the application package, or we need to create one? + # Using "if not exists" should take care of either case. + console.step( + f"Checking if stage {stage_fqn} exists, or creating a new one if none exists." + ) + with sql_executor.use_role(role): + sql_executor._execute_query( # noqa SLF001 + f"create schema if not exists {package_name}.{stage_schema}" + ) + sql_executor._execute_query( # noqa SLF001 + f""" + create stage if not exists {stage_fqn} + encryption = (TYPE = 'SNOWFLAKE_SSE') + DIRECTORY = (ENABLE = TRUE)""" + ) + + # Perform a diff operation and display results to the user for informational purposes + if print_diff: + console.step( + "Performing a diff between the Snowflake stage and your local deploy_root ('%s') directory." + % deploy_root.resolve() + ) + diff: DiffResult = compute_stage_diff(deploy_root, stage_fqn) + + if local_paths_to_sync: + # Deploying specific files/directories + resolved_paths_to_sync = [ + resolve_without_follow(p) for p in local_paths_to_sync + ] + if not recursive: + verify_no_directories(resolved_paths_to_sync) + + deploy_paths_to_sync = [] + for resolved_path in resolved_paths_to_sync: + verify_exists(resolved_path) + deploy_paths = bundle_map.to_deploy_paths(resolved_path) + if not deploy_paths: + if resolved_path.is_dir() and recursive: + # No direct artifact mapping found for this path. Check to see + # if there are subpaths of this directory that are matches. We + # loop over sources because it's likely a much smaller list + # than the project directory. + for src in bundle_map.all_sources(absolute=True): + if resolved_path in src.parents: + # There is a source that contains this path, get its dest path(s) + deploy_paths.extend(bundle_map.to_deploy_paths(src)) + + if not deploy_paths: + raise ClickException(f"No artifact found for {resolved_path}") + deploy_paths_to_sync.extend(deploy_paths) + + stage_paths_to_sync = _get_stage_paths_to_sync( + deploy_paths_to_sync, resolve_without_follow(deploy_root) + ) + diff = preserve_from_diff(diff, stage_paths_to_sync) + else: + # Full deploy + if not recursive: + verify_no_directories(deploy_root.resolve().iterdir()) + + if not prune: + files_not_removed = [str(path) for path in diff.only_on_stage] + diff.only_on_stage = [] + + if len(files_not_removed) > 0: + files_not_removed_str = "\n".join(files_not_removed) + console.warning( + f"The following files exist only on the stage:\n{files_not_removed_str}\n\nUse the --prune flag to delete them from the stage." + ) + + if print_diff: + print_diff_to_console(diff, bundle_map) + + # Upload diff-ed files to application package stage + if diff.has_changes(): + console.step( + "Updating the Snowflake stage from your local %s directory." + % deploy_root.resolve(), + ) + sync_local_diff_with_stage( + role=role, + deploy_root_path=deploy_root, + diff_result=diff, + stage_fqn=stage_fqn, + ) + return diff