Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make manifest optional in application package pdfV2 #1859

Merged
merged 4 commits into from
Nov 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion RELEASE-NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
## Backward incompatibility

## Deprecations

* `manifest` field of `application package` entity is now optional. This field does not have any functionality.
## New additions
* Added `--retain-comments` option to `snow sql` command to allow passing comments to Snowflake.
* Added `--replace` and `--if-not-exists` options to `snow object create` command.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,9 @@ class ApplicationPackageEntityModel(EntityModelBase):
title="Distribution of the application package created by the Snowflake CLI",
default="internal",
)
manifest: str = Field(
title="Path to manifest.yml",
manifest: Optional[str] = Field(
title="Path to manifest.yml. Unused and deprecated starting with Snowflake CLI 3.2",
default="",
)

@field_validator("identifier")
Expand Down
40 changes: 1 addition & 39 deletions src/snowflake/cli/api/project/definition_conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@

from click import ClickException
from snowflake.cli._plugins.nativeapp.artifacts import (
BundleMap,
bundle_artifacts,
)
from snowflake.cli._plugins.nativeapp.bundle_context import BundleContext
Expand Down Expand Up @@ -118,9 +117,7 @@ def convert_project_definition_to_v2(
else {}
)
native_app_data, native_app_template_context = (
convert_native_app_to_v2_data(
project_root, definition_v1.native_app, template_context
)
convert_native_app_to_v2_data(definition_v1.native_app, template_context)
if definition_v1.native_app
else ({}, {})
)
Expand Down Expand Up @@ -262,7 +259,6 @@ def convert_streamlit_to_v2_data(streamlit: Streamlit) -> Dict[str, Any]:


def convert_native_app_to_v2_data(
project_root: Path,
native_app: NativeApp,
template_context: Optional[Dict[str, Any]] = None,
) -> tuple[dict[str, Any], dict[str, Any]]:
Expand All @@ -276,39 +272,6 @@ def _make_meta(obj: Application | Package):
meta["post_deploy"] = obj.post_deploy
return meta

def _find_manifest():
# We don't know which file in the project directory is the actual manifest,
# and we can't iterate through the artifacts property since the src can contain
# glob patterns. The simplest solution is to bundle the app and find the
# manifest file from the resultant BundleMap, since the bundle process ensures
# that only a single source path can map to the corresponding destination path
bundle_map = BundleMap(
project_root=project_root, deploy_root=project_root / native_app.deploy_root
)
for artifact in native_app.artifacts:
bundle_map.add(artifact)

manifest_path = next(
(
src
for src, dest in bundle_map.all_mappings(
absolute=True, expand_directories=True
)
if dest.name == "manifest.yml"
),
None,
)
if not manifest_path:
# The manifest field is required, so we can't gracefully handle it being missing
raise ClickException(
"manifest.yml file not found in any Native App artifact sources, "
"unable to perform migration"
)

# Use a POSIX path to be consistent with other migrated fields
# which use POSIX paths as default values
return manifest_path.relative_to(project_root).as_posix()

package_entity_name = "pkg"
if (
native_app.package
Expand All @@ -324,7 +287,6 @@ def _find_manifest():
package = {
"type": "application package",
"identifier": package_identifier,
"manifest": _find_manifest(),
"artifacts": native_app.artifacts,
}

Expand Down
123 changes: 121 additions & 2 deletions tests/helpers/__snapshots__/test_v1_to_v2.ambr
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,127 @@

'''
# ---
# name: test_migration_native_app_no_artifacts
'''
definition_version: '2'
entities:
procedureName:
imports: []
external_access_integrations: []
secrets: {}
meta:
use_mixins:
- snowpark_shared
identifier:
name: procedureName
handler: hello
returns: string
signature:
- name: name
type: string
stage: dev_deployment
artifacts:
- src: app
dest: my_snowpark_project
type: procedure
execute_as_caller: false
func1:
imports: []
external_access_integrations: []
secrets: {}
meta:
use_mixins:
- snowpark_shared
identifier:
name: func1
handler: app.func1_handler
returns: string
signature:
- name: a
type: string
default: default value
- name: b
type: variant
runtime: '3.1'
stage: dev_deployment
artifacts:
- src: app
dest: my_snowpark_project
type: function
test_streamlit:
identifier:
name: test_streamlit
type: streamlit
title: My Fancy Streamlit
query_warehouse: test_warehouse
main_file: streamlit_app.py
pages_dir: None
stage: streamlit
artifacts:
- streamlit_app.py
- environment.yml
- pages
pkg:
meta:
role: pkg_role
identifier: <% fn.concat_ids('myapp', '_pkg_', fn.sanitize_id(fn.get_username('unknown_user')) | lower) %>
type: application package
artifacts: []
app:
identifier: myapp_app
type: application
from:
target: pkg
mixins:
snowpark_shared:
stage: dev_deployment
artifacts:
- src: app/
dest: my_snowpark_project

'''
# ---
# name: test_migration_native_app_no_artifacts.1
'''
definition_version: 1
native_app:
application:
name: myapp_app
artifacts: []
name: myapp
package:
role: pkg_role
snowpark:
functions:
- handler: app.func1_handler
name: func1
returns: string
runtime: 3.1
signature:
- default: default value
name: a
type: string
- name: b
type: variant
procedures:
- handler: hello
name: procedureName
returns: string
signature:
- name: name
type: string
project_name: my_snowpark_project
src: app/
stage_name: dev_deployment
streamlit:
main_file: streamlit_app.py
name: test_streamlit
query_warehouse: test_warehouse
stage: streamlit
title: My Fancy Streamlit

'''
# ---
# name: test_migrations_with_all_app_entities
'''
definition_version: '2'
Expand Down Expand Up @@ -196,7 +317,6 @@
stage: app_src.my_stage
scratch_stage: app_src.my_scratch
distribution: external
manifest: app/manifest.yml
app:
meta:
warehouse: app_wh
Expand Down Expand Up @@ -316,7 +436,6 @@
artifacts:
- src: app/*
dest: ./
manifest: app/manifest.yml
app:
identifier: myapp_app
type: application
Expand Down
40 changes: 7 additions & 33 deletions tests/helpers/test_v1_to_v2.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import logging
import shutil
from pathlib import Path
from textwrap import dedent

Expand Down Expand Up @@ -38,36 +37,10 @@ def test_migrations_with_all_app_entities(
assert Path("snowflake_V1.yml").read_text() == os_agnostic_snapshot


def test_migration_native_app_nested_manifest(runner, project_directory):
with project_directory("migration_multiple_entities") as project_dir:
with open("snowflake.yml", "r+") as snowflake_yml:
pdf = yaml.safe_load(snowflake_yml)
pdf["native_app"]["artifacts"] = [
dict(src="nested/nested/app", dest="nested/app/")
]
snowflake_yml.seek(0)
yaml.safe_dump(pdf, snowflake_yml)
snowflake_yml.truncate()

shutil.move("app", "nested/nested/app")
result = runner.invoke(["helpers", "v1-to-v2"])
assert result.exit_code == 0
with open("snowflake.yml") as f:
pdf = yaml.safe_load(f)
assert (
pdf["entities"]["pkg"]["manifest"] == "nested/nested/app/manifest.yml"
)


def test_migration_native_app_missing_manifest(runner, project_directory):
with project_directory("migration_multiple_entities") as project_dir:
(project_dir / "app" / "manifest.yml").unlink()
result = runner.invoke(["helpers", "v1-to-v2"])
assert result.exit_code == 1
assert "manifest.yml file not found" in result.output


def test_migration_native_app_no_artifacts(runner, project_directory):
# Migration of app without artifacts shouldn't fail
def test_migration_native_app_no_artifacts(
runner, project_directory, os_agnostic_snapshot
):
with project_directory("migration_multiple_entities") as project_dir:
with (project_dir / "snowflake.yml").open("r+") as snowflake_yml:
pdf = yaml.safe_load(snowflake_yml)
Expand All @@ -76,8 +49,9 @@ def test_migration_native_app_no_artifacts(runner, project_directory):
yaml.safe_dump(pdf, snowflake_yml)
snowflake_yml.truncate()
result = runner.invoke(["helpers", "v1-to-v2"])
assert result.exit_code == 1
assert "manifest.yml file not found" in result.output
assert result.exit_code == 0
assert Path("snowflake.yml").read_text() == os_agnostic_snapshot
assert Path("snowflake_V1.yml").read_text() == os_agnostic_snapshot


def test_migration_native_app_package_scripts(runner, project_directory):
Expand Down
1 change: 0 additions & 1 deletion tests/project/test_project_definition_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@
{"entities": {"pkg": {"type": "application package"}}},
[
"missing the following field: 'entities.pkg.application package.artifacts'",
"missing the following field: 'entities.pkg.application package.manifest'",
],
],
[
Expand Down