Fixes for package creation and pip installs (#781)

* Added dependencies check to venv

* Problem

* Problem2

* Fixed problem

* Cleanup

* Corrected github installs

* Corrected lib path

* added release notes

* fixes

* fixes

* fixes

* fixes

* Added native libraries option

* Added native libraries option

* Snapshot update

* Snapshot update

* Fixes

* After merge fixes

* Minor

* fixes

* Updates

* Performance update

* Test fix

* windows fix

* windows fix

* windows fix

* windows fix

* windows fix

* windows fix

* windows fix

* windows fix

* fixed tests

* fixed tests

* Fixes after review

* Fixes after review

* Fixes

* Fixes

* Fixes

* Fixes

* Fixes

* Fixes

* Fixes

* Fixes

* Replaced deprecated flags

* Replaced deprecated flags

* Replaced deprecated flags

* Replaced deprecated flags

* Fix for encoding

* Fix for encoding

* Fix for encoding

* Fix for encoding

* Fix for encoding

* Started cleaning

* Cleanup

.pre-commit-config.yaml

@@ -40,7 +40,8 @@ repos:
           ^src/snowflake/cli/app/dev/.*$|
           ^src/snowflake/cli/templates/.*$|
           ^src/snowflake/cli/api/utils/rendering.py$|
-          ^src/snowflake/cli/plugins/spcs/common.py$
+          ^src/snowflake/cli/plugins/spcs/common.py$|
+          ^src/snowflake/cli/plugins/snowpark/venv.py$
       - id: check-app-imports-in-api
         language: pygrep
         name: "No top level cli.app imports in cli.api"

RELEASE-NOTES.md

@@ -28,6 +28,9 @@
   * Added `create` command for `image-repository`.
   * Added `status`, `set (property)`, `unset (property)`, `suspend` and `resume` commands for `compute-pool`.
   * Added `set (property)`, `unset (property)`,`upgrade` and `list-endpoints` commands for `service`.
+* You can now use github repo link in `snow snowpark package create` to prepare your code for upload
+* Added `allow-native-libraries` option to `snow snowpark package create` command
+* Added alias `--install-from-pip` for `-y` option in `snow snowpark package create` command
 * Connections parameters are also supported by generic environment variables:
   * `SNOWFLAKE_ACCOUNT`
   * `SNOWFLAKE_USER`
@@ -49,6 +52,7 @@
 ## Fixes and improvements
 * Restricted permissions of automatically created files
 * Fixed bug where `spcs service create` would not throw error if service with specified name already exists.
+* Improved package lookup, to avoid unnecessary uploads
 * Logging into the file by default (INFO level)
 * Added validation that service, compute pool, and image repository names are unqualified identifiers.
 * `spcs service` commands now accept qualified names.

src/snowflake/cli/plugins/snowpark/models.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
+import re
 from dataclasses import dataclass
 from enum import Enum
 from typing import List
 
-from requirements.requirement import Requirement
+from requirements import requirement
 
 
 class PypiOption(Enum):
@@ -13,6 +14,35 @@ class PypiOption(Enum):
     ASK = "ask"
 
 
+class RequirementType(Enum):
+    FILE = "file"
+    PACKAGE = "package"
+
+
+class Requirement(requirement.Requirement):
+    extra_pattern = re.compile("'([^']*)'")
+
+    @classmethod
+    def parse_line(cls, line: str) -> Requirement:
+        if len(line_elements := line.split(";")) > 1:
+            line = line_elements[0]
+        result = super().parse_line(line)
+
+        if len(line_elements) > 1:
+            for element in line_elements[1:]:
+                if "extra" in element and (extras := cls.extra_pattern.search(element)):
+                    result.extras.extend(extras.groups())
+
+        if result.uri and not result.name:
+            result.name = get_package_name(result.uri)
+
+        return result
+
+    @classmethod
+    def _look_for_specifier(cls, specifier: str, line: str):
+        return re.search(cls.specifier_pattern.format(specifier), line)
+
+
 @dataclass
 class SplitRequirements:
     """A dataclass to hold the results of parsing requirements files and dividing them into
@@ -30,3 +60,38 @@ class RequirementWithFiles:
 
     requirement: Requirement
     files: List[str]
+
+
+@dataclass
+class RequirementWithFilesAndDeps(RequirementWithFiles):
+    dependencies: List[str]
+
+    def to_requirement_with_files(self):
+        return RequirementWithFiles(self.requirement, self.files)
+
+
+def get_package_name(name: str) -> str:
+    if name.lower().startswith(("git+", "http")):
+        pattern = re.compile(r"github\.com\/[^\/]+\/([^\/][^.@$/]+)")
+        if match := pattern.search(name):
+            return match.group(1)
+        else:
+            return name
+
+    elif name.endswith(".zip"):
+        return name.replace(".zip", "")
+    else:
+        return name
+
+
+pip_failed_msg = (
+    "pip failed with return code {}."
+    "If pip is installed correctly, this may mean you`re trying to install a package"
+    "that isn't compatible with the host architecture -"
+    "and generally means it has native libraries."
+)
+second_chance_msg = (
+    "Good news! The following package dependencies can be"
+    "imported directly from Anaconda, and will be excluded from"
+    "the zip: {}"
+)

src/snowflake/cli/plugins/snowpark/package/commands.py

@@ -6,6 +6,7 @@
 import typer
 from snowflake.cli.api.commands.snow_typer import SnowTyper
 from snowflake.cli.api.output.types import CommandResult, MessageResult
+from snowflake.cli.plugins.snowpark.models import PypiOption
 from snowflake.cli.plugins.snowpark.package.manager import (
     cleanup_after_install,
     create,
@@ -17,6 +18,7 @@
     NotInAnaconda,
     RequiresPackages,
 )
+from snowflake.cli.plugins.snowpark.snowpark_shared import PackageNativeLibrariesOption
 
 app = SnowTyper(
     name="package",
@@ -45,6 +47,7 @@ def package_lookup(
     name: str = typer.Argument(..., help="Name of the package."),
     install_packages: bool = install_option,
     deprecated_install_option: bool = deprecated_install_option,
+    allow_native_libraries: PypiOption = PackageNativeLibrariesOption,
     **options,
 ) -> CommandResult:
     """
@@ -55,7 +58,11 @@ def package_lookup(
     if deprecated_install_option:
         install_packages = deprecated_install_option
 
-    lookup_result = lookup(name=name, install_packages=install_packages)
+    lookup_result = lookup(
+        name=name,
+        install_packages=install_packages,
+        allow_native_libraries=allow_native_libraries,
+    )
     return MessageResult(lookup_result.message)
 
 
@@ -97,6 +104,7 @@ def package_create(
     ),
     install_packages: bool = install_option,
     deprecated_install_option: bool = deprecated_install_option,
+    allow_native_libraries: PypiOption = PackageNativeLibrariesOption,
     **options,
 ) -> CommandResult:
     """
@@ -105,13 +113,14 @@ def package_create(
     if deprecated_install_option:
         install_packages = deprecated_install_option
 
-    if (
-        type(lookup_result := lookup(name=name, install_packages=install_packages))
-        in [
-            NotInAnaconda,
-            RequiresPackages,
-        ]
-        and type(creation_result := create(name)) == CreatedSuccessfully
+    lookup_result = lookup(
+        name=name,
+        install_packages=install_packages,
+        allow_native_libraries=allow_native_libraries,
+    )
+
+    if isinstance(lookup_result, (NotInAnaconda, RequiresPackages)) and isinstance(
+        creation_result := create(name), CreatedSuccessfully
     ):
         message = creation_result.message
         if type(lookup_result) == RequiresPackages:

src/snowflake/cli/plugins/snowpark/package/manager.py

@@ -5,12 +5,16 @@
 from functools import wraps
 from pathlib import Path
 
-from requirements.requirement import Requirement
 from snowflake.cli.api.constants import PACKAGES_DIR
 from snowflake.cli.api.secure_path import SecurePath
 from snowflake.cli.plugins.object.stage.manager import StageManager
 from snowflake.cli.plugins.snowpark import package_utils
-from snowflake.cli.plugins.snowpark.models import SplitRequirements
+from snowflake.cli.plugins.snowpark.models import (
+    PypiOption,
+    Requirement,
+    SplitRequirements,
+    get_package_name,
+)
 from snowflake.cli.plugins.snowpark.package.utils import (
     CreatedSuccessfully,
     InAnaconda,
@@ -25,15 +29,20 @@
 log = logging.getLogger(__name__)
 
 
-def lookup(name: str, install_packages: bool) -> LookupResult:
+def lookup(
+    name: str, install_packages: bool, allow_native_libraries: PypiOption
+) -> LookupResult:
 
     package_response = package_utils.parse_anaconda_packages([Requirement.parse(name)])
 
     if package_response.snowflake and not package_response.other:
         return InAnaconda(package_response, name)
     elif install_packages:
         status, result = package_utils.install_packages(
-            perform_anaconda_check=True, package_name=name, file_name=None
+            perform_anaconda_check=True,
+            package_name=name,
+            file_name=None,
+            allow_native_libraries=allow_native_libraries,
         )
 
         if status:
@@ -65,7 +74,7 @@ def upload(file: Path, stage: str, overwrite: bool):
 
 
 def create(zip_name: str):
-    file_name = zip_name if zip_name.endswith(".zip") else f"{zip_name}.zip"
+    file_name = f"{get_package_name(zip_name)}.zip"
     zip_dir(dest_zip=Path(file_name), source=Path.cwd() / ".packages")
 
     if os.path.exists(file_name):

src/snowflake/cli/plugins/snowpark/package/utils.py

@@ -4,9 +4,8 @@
 from pathlib import Path
 from typing import List
 
-from requirements.requirement import Requirement
 from snowflake.cli.api.secure_path import SecurePath
-from snowflake.cli.plugins.snowpark.models import SplitRequirements
+from snowflake.cli.plugins.snowpark.models import Requirement, SplitRequirements
 
 
 @dataclass