New dependencies handling

.github/workflows/test.yml

@@ -38,6 +38,7 @@ jobs:
     - name: Unit tests
       run: |
         pytest tests
+        pytest --doctest-modules komodo
 
     - name: Lint examples
       run: |

ci/repository.yml

@@ -3,16 +3,10 @@ numpy:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - setuptools
-      - python
   1.23.5:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - setuptools
-      - python
 
 python:
   3-builtin:
@@ -26,14 +20,9 @@ setuptools:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - wheel
-      - python
 
 wheel:
   0.42.0:
     source: pypi
     make: pip
     maintainer: ci
-    depends:
-      - python

komodo/build.py

@@ -4,9 +4,8 @@
 import os
 import re
 import stat
-import sys
 from pathlib import Path
-from typing import Dict, List, Optional, Set
+from typing import Dict
 
 import requests
 
@@ -17,42 +16,6 @@
 )
 from komodo.shell import pushd, shell
 
-
-def full_dfs(
-    all_packages: Dict[str, str],
-    repo: Dict[str, Dict],
-    packages_to_check: Optional[List[str]] = None,
-) -> List[str]:
-    if packages_to_check is None:
-        packages_to_check = list(all_packages.keys())
-    all_packages_set = set(all_packages)
-
-    def dfs(package_name: str, pkg_order: List[str], visited: Set[str]):
-        if package_name in visited:
-            return
-        visited.add(package_name)
-        ver = all_packages[package_name]
-        dependencies = repo[package_name][ver].get("depends", [])
-        missing_depends = set(dependencies) - all_packages_set
-        if missing_depends:
-            print(
-                "error: "
-                + ",".join(missing_depends)
-                + f" required as dependency for {package_name}, is not in distribution",
-                file=sys.stderr,
-            )
-            sys.exit(1)
-        for dep in dependencies:
-            dfs(dep, pkg_order, visited)
-        pkg_order.append(package_name)
-
-    visited = set()
-    pkg_order = []
-    for package in packages_to_check:
-        dfs(package, pkg_order, visited)
-    return pkg_order
-
-
 # When running cmake we pass the option -DDEST_PREFIX=fakeroot, this is an
 # absolute hack to be able to build opm-common and sunbeam with the ~fakeroot
 # implementation used by komodo.
@@ -258,8 +221,7 @@ def make(
     pip="pip",
     fakeroot=".",
 ):
-    pkgorder = full_dfs(pkgs, repo)
-    assert len(set(pkgorder)) == len(pkgs)
+    pkgorder = list(pkgs.keys())
     fakeprefix = fakeroot + prefix
     shell(["mkdir -p", fakeprefix])
     prefix = os.path.abspath(prefix)

komodo/check_unused_package.py

@@ -4,33 +4,35 @@
 import os
 import sys
 
-from komodo.build import full_dfs
 from komodo.prettier import load_yaml
 from komodo.yaml_file_types import ReleaseFile, RepositoryFile
 
+from .pypi_dependencies import PypiDependencies
+
 
 def check_for_unused_package(
     release_file: ReleaseFile, package_status_file: str, repository: RepositoryFile
 ):
     package_status = load_yaml(package_status_file)
-    private_packages = [
-        pkg
-        for pkg in release_file.content
-        if package_status[pkg]["visibility"] == "private"
-    ]
     public_and_plugin_packages = [
-        pkg
-        for pkg in release_file.content
+        (pkg, version)
+        for pkg, version in release_file.content.items()
         if package_status[pkg]["visibility"] in ("public", "private-plugin")
     ]
-    public_and_plugin_dependencies = full_dfs(
-        release_file.content,
-        repository.content,
-        public_and_plugin_packages,
-    )
-    diff_packages = set(private_packages).difference(
-        set(public_and_plugin_dependencies)
-    )
+    python_version = release_file.content["python"]
+    # For pypi we need to drop bugfix version '3.8.6-builtin' ->
+    python_version = python_version[: python_version.rindex(".")]
+    dependencies = PypiDependencies(release_file.content, python_version=python_version)
+    for name, version in release_file.content.items():
+        metadata = repository.content.get(name, {}).get(version, {})
+        if metadata.get("source") != "pypi":
+            dependencies.add_user_specified(name, metadata.get("depends", []))
+    dependencies.check(public_and_plugin_packages)
+    diff_packages = set(
+        pkg
+        for pkg in release_file.content
+        if package_status[pkg]["visibility"] == "private"
+    ).difference(dependencies.used_packages)
     if diff_packages:
         print(
             f"The following {len(diff_packages)} private packages are not dependencies of any public or private-plugin packages:"

komodo/lint.py

@@ -8,7 +8,8 @@
 
 from packaging.version import parse
 
-from komodo.yaml_file_types import KomodoException, ReleaseFile, RepositoryFile
+from .pypi_dependencies import PypiDependencies
+from .yaml_file_types import KomodoException, ReleaseFile, RepositoryFile
 
 KomodoError = namedtuple(
     "KomodoError",
@@ -66,8 +67,12 @@ def lint_version_numbers(package, version, repo):
     return None
 
 
-def lint(release_file: ReleaseFile, repository_file: RepositoryFile):
-    maintainers, deps, versions = [], [], []
+def lint(
+    release_file: ReleaseFile,
+    repository_file: RepositoryFile,
+    check_dependencies: bool = False,
+) -> Report:
+    maintainers, versions = [], []
     for package_name, package_version in release_file.content.items():
         try:
             lint_maintainer = repository_file.lint_maintainer(
@@ -84,27 +89,43 @@ def lint(release_file: ReleaseFile, repository_file: RepositoryFile):
             )
             if lint_version_number:
                 versions.append(lint_version_number)
-            missing = []
-            repository_file_package_version_data = repository_file.content.get(
-                package_name,
-            ).get(package_version)
-            for dependency in repository_file_package_version_data.get("depends", []):
-                if dependency not in release_file.content:
-                    missing.append(dependency)
-            if missing:
-                deps.append(
-                    _komodo_error(
-                        package=package_name,
-                        version=package_version,
-                        depends=missing,
-                        err=(
-                            f"{MISSING_DEPENDENCY} for {package_name} {package_version}"
-                        ),
-                    ),
-                )
         except KomodoException as komodo_exception:
             maintainers.append(komodo_exception.error)
 
+    if check_dependencies:
+        pypi_dependencies = {
+            name: version
+            for name, version in release_file.content.items()
+            if repository_file.content.get(name, {}).get(version, {}).get("source")
+            == "pypi"
+        }
+
+        python_version = release_file.content["python"]
+        # For pypi we need to drop bugfix version '3.8.6-builtin' ->
+        python_version = python_version[: python_version.rindex(".")]
+        dependencies = PypiDependencies(
+            pypi_dependencies, python_version=python_version
+        )
+        for name, version in release_file.content.items():
+            if "depends" in repository_file.content.get(name, {}).get(version, {}):
+                dependencies.add_user_specified(
+                    name, repository_file.content[name][version]["depends"]
+                )
+
+        dependencies.check()
+        if dependencies.failed_requirements:
+            deps = [
+                _komodo_error(
+                    err="Failed requirements ",
+                    depends=[str(r) for r in dependencies.failed_requirements],
+                )
+            ]
+        else:
+            deps = []
+        dependencies.dump_cache()
+    else:
+        deps = []
+
     return Report(
         release_name=[],
         maintainers=maintainers,
@@ -135,31 +156,38 @@ def get_args():
         dest="loglevel",
         const=logging.INFO,
     )
+    parser.add_argument(
+        "--check-pypi-dependencies",
+        dest="check_pypi_dependencies",
+        help="Checks package metadata",
+        action="store_true",
+        default=False,
+    )
     return parser.parse_args()
 
 
 def lint_main():
     args = get_args()
     logging.basicConfig(format="%(message)s", level=args.loglevel)
 
-    try:
-        report = lint(args.packagefile, args.repofile)
-        maintainers, deps, versions = (
-            report.maintainers,
-            report.dependencies,
-            report.versions,
-        )
-    except ValueError as err:
-        sys.exit(str(err))
+    report = lint(
+        args.packagefile, args.repofile, check_dependencies=args.check_pypi_dependencies
+    )
+    maintainers, deps, versions = (
+        report.maintainers,
+        report.dependencies,
+        report.versions,
+    )
     print(f"{len(maintainers)} packages")
     if not any(err.err for err in maintainers + deps + versions):
         print("No errors found")
         sys.exit(0)
 
     for err in maintainers + deps + versions:
         if err.err:
-            dep = f": {', '.join(err.depends)}" if err.depends else ""
-            print(f"{err.err}{dep}")
+            print(f"{err.err}")
+            if err.depends:
+                print("\n  ".join(err.depends))
 
     if not any(err.err for err in maintainers + deps):
         sys.exit(0)  # currently we allow erronous version numbers