Skip to content

Commit

Permalink
sync_deploy_root_with_stage
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-gbloom
sfc-gh-gbloom committed Aug 14, 2024
1 parent 0cbc374 commit 88e0591
Show file tree
Hide file tree
Showing 2 changed files with 189 additions and 129 deletions.
158 changes: 30 additions & 128 deletions src/snowflake/cli/_plugins/nativeapp/manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
from __future__ import annotations

import json
import os
import time
from abc import ABC, abstractmethod
from contextlib import contextmanager
Expand All @@ -31,7 +30,6 @@
from snowflake.cli._plugins.nativeapp.artifacts import (
BundleMap,
build_bundle,
resolve_without_follow,
)
from snowflake.cli._plugins.nativeapp.codegen.compiler import (
NativeAppCompiler,
Expand All @@ -49,24 +47,18 @@
from snowflake.cli._plugins.nativeapp.project_model import (
NativeAppProjectModel,
)
from snowflake.cli._plugins.nativeapp.utils import verify_exists, verify_no_directories
from snowflake.cli._plugins.stage.diff import (
DiffResult,
StagePath,
compute_stage_diff,
preserve_from_diff,
sync_local_diff_with_stage,
to_stage_path,
)
from snowflake.cli._plugins.stage.manager import StageManager
from snowflake.cli._plugins.stage.utils import print_diff_to_console
from snowflake.cli.api.cli_global_context import get_cli_context
from snowflake.cli.api.console import cli_console as cc
from snowflake.cli.api.entities.application_package_entity import (
ApplicationPackageEntity,
)
from snowflake.cli.api.entities.utils import (
generic_sql_error_handler,
sync_deploy_root_with_stage,
)
from snowflake.cli.api.errno import (
DOES_NOT_EXIST_OR_NOT_AUTHORIZED,
Expand All @@ -90,23 +82,23 @@
ApplicationOwnedObject = TypedDict("ApplicationOwnedObject", {"name": str, "type": str})


def _get_stage_paths_to_sync(
local_paths_to_sync: List[Path], deploy_root: Path
) -> List[StagePath]:
"""
Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root.
"""
# def _get_stage_paths_to_sync(
# local_paths_to_sync: List[Path], deploy_root: Path
# ) -> List[StagePath]:
# """
# Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root.
# """

stage_paths = []
for path in local_paths_to_sync:
if path.is_dir():
for current_dir, _dirs, files in os.walk(path):
for file in files:
deploy_path = Path(current_dir, file).relative_to(deploy_root)
stage_paths.append(to_stage_path(deploy_path))
else:
stage_paths.append(to_stage_path(path.relative_to(deploy_root)))
return stage_paths
# stage_paths = []
# for path in local_paths_to_sync:
# if path.is_dir():
# for current_dir, _dirs, files in os.walk(path):
# for file in files:
# deploy_path = Path(current_dir, file).relative_to(deploy_root)
# stage_paths.append(to_stage_path(deploy_path))
# else:
# stage_paths.append(to_stage_path(path.relative_to(deploy_root)))
# return stage_paths


class NativeAppCommandProcessor(ABC):
Expand Down Expand Up @@ -288,110 +280,20 @@ def sync_deploy_root_with_stage(
local_paths_to_sync: List[Path] | None = None,
print_diff: bool = True,
) -> DiffResult:
"""
Ensures that the files on our remote stage match the artifacts we have in
the local filesystem.
Args:
bundle_map (BundleMap): The artifact mapping computed by the `build_bundle` function.
role (str): The name of the role to use for queries and commands.
prune (bool): Whether to prune artifacts from the stage that don't exist locally.
recursive (bool): Whether to traverse directories recursively.
stage_fqn (str): The name of the stage to diff against and upload to.
local_paths_to_sync (List[Path], optional): List of local paths to sync. Defaults to None to sync all
local paths. Note that providing an empty list here is equivalent to None.
print_diff (bool): Whether to print the diff between the local files and the remote stage. Defaults to True
Returns:
A `DiffResult` instance describing the changes that were performed.
"""

# Does a stage already exist within the application package, or we need to create one?
# Using "if not exists" should take care of either case.
cc.step(
f"Checking if stage {stage_fqn} exists, or creating a new one if none exists."
return sync_deploy_root_with_stage(
sql_executor=self,
console=cc,
deploy_root=self.deploy_root,
package_name=self.package_name,
stage_schema=self.stage_schema,
bundle_map=bundle_map,
role=role,
prune=prune,
recursive=recursive,
stage_fqn=stage_fqn,
local_paths_to_sync=local_paths_to_sync,
print_diff=print_diff,
)
with self.use_role(role):
self._execute_query(
f"create schema if not exists {self.package_name}.{self.stage_schema}"
)
self._execute_query(
f"""
create stage if not exists {stage_fqn}
encryption = (TYPE = 'SNOWFLAKE_SSE')
DIRECTORY = (ENABLE = TRUE)"""
)

# Perform a diff operation and display results to the user for informational purposes
if print_diff:
cc.step(
"Performing a diff between the Snowflake stage and your local deploy_root ('%s') directory."
% self.deploy_root.resolve()
)
diff: DiffResult = compute_stage_diff(self.deploy_root, stage_fqn)

if local_paths_to_sync:
# Deploying specific files/directories
resolved_paths_to_sync = [
resolve_without_follow(p) for p in local_paths_to_sync
]
if not recursive:
verify_no_directories(resolved_paths_to_sync)

deploy_paths_to_sync = []
for resolved_path in resolved_paths_to_sync:
verify_exists(resolved_path)
deploy_paths = bundle_map.to_deploy_paths(resolved_path)
if not deploy_paths:
if resolved_path.is_dir() and recursive:
# No direct artifact mapping found for this path. Check to see
# if there are subpaths of this directory that are matches. We
# loop over sources because it's likely a much smaller list
# than the project directory.
for src in bundle_map.all_sources(absolute=True):
if resolved_path in src.parents:
# There is a source that contains this path, get its dest path(s)
deploy_paths.extend(bundle_map.to_deploy_paths(src))

if not deploy_paths:
raise ClickException(f"No artifact found for {resolved_path}")
deploy_paths_to_sync.extend(deploy_paths)

stage_paths_to_sync = _get_stage_paths_to_sync(
deploy_paths_to_sync, resolve_without_follow(self.deploy_root)
)
diff = preserve_from_diff(diff, stage_paths_to_sync)
else:
# Full deploy
if not recursive:
verify_no_directories(self.deploy_root.resolve().iterdir())

if not prune:
files_not_removed = [str(path) for path in diff.only_on_stage]
diff.only_on_stage = []

if len(files_not_removed) > 0:
files_not_removed_str = "\n".join(files_not_removed)
cc.warning(
f"The following files exist only on the stage:\n{files_not_removed_str}\n\nUse the --prune flag to delete them from the stage."
)

if print_diff:
print_diff_to_console(diff, bundle_map)

# Upload diff-ed files to application package stage
if diff.has_changes():
cc.step(
"Updating the Snowflake stage from your local %s directory."
% self.deploy_root.resolve(),
)
sync_local_diff_with_stage(
role=role,
deploy_root_path=self.deploy_root,
diff_result=diff,
stage_fqn=stage_fqn,
)
return diff

def get_existing_app_info(self) -> Optional[dict]:
"""
Expand Down
160 changes: 159 additions & 1 deletion src/snowflake/cli/api/entities/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,32 @@
import os
from pathlib import Path
from textwrap import dedent
from typing import NoReturn, Optional
from typing import List, NoReturn, Optional

from click import ClickException
from snowflake.cli._plugins.nativeapp.artifacts import (
BundleMap,
resolve_without_follow,
)
from snowflake.cli._plugins.nativeapp.constants import OWNER_COL
from snowflake.cli._plugins.nativeapp.exceptions import UnexpectedOwnerError
from snowflake.cli._plugins.nativeapp.utils import verify_exists, verify_no_directories
from snowflake.cli._plugins.stage.diff import (
DiffResult,
StagePath,
compute_stage_diff,
preserve_from_diff,
sync_local_diff_with_stage,
to_stage_path,
)
from snowflake.cli._plugins.stage.utils import print_diff_to_console
from snowflake.cli.api.console.abc import AbstractConsole
from snowflake.cli.api.errno import (
DOES_NOT_EXIST_OR_CANNOT_BE_PERFORMED,
NO_WAREHOUSE_SELECTED_IN_SESSION,
)
from snowflake.cli.api.project.util import unquote_identifier
from snowflake.cli.api.sql_execution import SqlExecutionMixin
from snowflake.connector import ProgrammingError


Expand Down Expand Up @@ -57,3 +76,142 @@ def ensure_correct_owner(row: dict, role: str, obj_name: str) -> None:
].upper() # Because unquote_identifier() always returns uppercase str
if actual_owner != unquote_identifier(role):
raise UnexpectedOwnerError(obj_name, role, actual_owner)


def _get_stage_paths_to_sync(
local_paths_to_sync: List[Path], deploy_root: Path
) -> List[StagePath]:
"""
Takes a list of paths (files and directories), returning a list of all files recursively relative to the deploy root.
"""

stage_paths = []
for path in local_paths_to_sync:
if path.is_dir():
for current_dir, _dirs, files in os.walk(path):
for file in files:
deploy_path = Path(current_dir, file).relative_to(deploy_root)
stage_paths.append(to_stage_path(deploy_path))
else:
stage_paths.append(to_stage_path(path.relative_to(deploy_root)))
return stage_paths


def sync_deploy_root_with_stage(
sql_executor: SqlExecutionMixin,
console: AbstractConsole,
deploy_root: Path,
package_name: str,
stage_schema: str,
bundle_map: BundleMap,
role: str,
prune: bool,
recursive: bool,
stage_fqn: str,
local_paths_to_sync: List[Path] | None = None,
print_diff: bool = True,
) -> DiffResult:
"""
Ensures that the files on our remote stage match the artifacts we have in
the local filesystem.
Args:
bundle_map (BundleMap): The artifact mapping computed by the `build_bundle` function.
role (str): The name of the role to use for queries and commands.
prune (bool): Whether to prune artifacts from the stage that don't exist locally.
recursive (bool): Whether to traverse directories recursively.
stage_fqn (str): The name of the stage to diff against and upload to.
local_paths_to_sync (List[Path], optional): List of local paths to sync. Defaults to None to sync all
local paths. Note that providing an empty list here is equivalent to None.
print_diff (bool): Whether to print the diff between the local files and the remote stage. Defaults to True
Returns:
A `DiffResult` instance describing the changes that were performed.
"""

# Does a stage already exist within the application package, or we need to create one?
# Using "if not exists" should take care of either case.
console.step(
f"Checking if stage {stage_fqn} exists, or creating a new one if none exists."
)
with sql_executor.use_role(role):
sql_executor._execute_query( # noqa SLF001
f"create schema if not exists {package_name}.{stage_schema}"
)
sql_executor._execute_query( # noqa SLF001
f"""
create stage if not exists {stage_fqn}
encryption = (TYPE = 'SNOWFLAKE_SSE')
DIRECTORY = (ENABLE = TRUE)"""
)

# Perform a diff operation and display results to the user for informational purposes
if print_diff:
console.step(
"Performing a diff between the Snowflake stage and your local deploy_root ('%s') directory."
% deploy_root.resolve()
)
diff: DiffResult = compute_stage_diff(deploy_root, stage_fqn)

if local_paths_to_sync:
# Deploying specific files/directories
resolved_paths_to_sync = [
resolve_without_follow(p) for p in local_paths_to_sync
]
if not recursive:
verify_no_directories(resolved_paths_to_sync)

deploy_paths_to_sync = []
for resolved_path in resolved_paths_to_sync:
verify_exists(resolved_path)
deploy_paths = bundle_map.to_deploy_paths(resolved_path)
if not deploy_paths:
if resolved_path.is_dir() and recursive:
# No direct artifact mapping found for this path. Check to see
# if there are subpaths of this directory that are matches. We
# loop over sources because it's likely a much smaller list
# than the project directory.
for src in bundle_map.all_sources(absolute=True):
if resolved_path in src.parents:
# There is a source that contains this path, get its dest path(s)
deploy_paths.extend(bundle_map.to_deploy_paths(src))

if not deploy_paths:
raise ClickException(f"No artifact found for {resolved_path}")
deploy_paths_to_sync.extend(deploy_paths)

stage_paths_to_sync = _get_stage_paths_to_sync(
deploy_paths_to_sync, resolve_without_follow(deploy_root)
)
diff = preserve_from_diff(diff, stage_paths_to_sync)
else:
# Full deploy
if not recursive:
verify_no_directories(deploy_root.resolve().iterdir())

if not prune:
files_not_removed = [str(path) for path in diff.only_on_stage]
diff.only_on_stage = []

if len(files_not_removed) > 0:
files_not_removed_str = "\n".join(files_not_removed)
console.warning(
f"The following files exist only on the stage:\n{files_not_removed_str}\n\nUse the --prune flag to delete them from the stage."
)

if print_diff:
print_diff_to_console(diff, bundle_map)

# Upload diff-ed files to application package stage
if diff.has_changes():
console.step(
"Updating the Snowflake stage from your local %s directory."
% deploy_root.resolve(),
)
sync_local_diff_with_stage(
role=role,
deploy_root_path=deploy_root,
diff_result=diff,
stage_fqn=stage_fqn,
)
return diff

0 comments on commit 88e0591

Please sign in to comment.