Make komodo RHEL agnostic

.github/workflows/test.yml

@@ -7,13 +7,13 @@ jobs:
     env:
       RELEASE: release-py3x.yml
 
-
     strategy:
       fail-fast: false
       matrix:
         python-version: ['3.8', '3.10', '3.11']
+        target-python-bin: ['/usr/bin/python3.8']
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
     - uses: actions/checkout@v4
@@ -30,27 +30,21 @@ jobs:
         sudo apt update
         sudo apt install csh
 
-    - name: Setup Komodo
+    - name: Install Komodo
       run: |
-        ./bootstrap.sh $(which python)
+        pip install .[dev]
 
     - name: Unit tests
       run: |
-        source boot/build-env/bin/activate
-        boot/kmd-env/bin/python -m pytest tests
+        pytest tests
 
     - name: Lint examples
       run: |
-        boot/kmd-env/bin/python -m komodo.lint examples/stable.yml examples/repository.yml
+        python -m komodo.lint examples/stable.yml examples/repository.yml
 
     - name: Full integration test
       run: |
-        ./run_kmd.sh \
-            ci/${RELEASE} \
-            ci/repository.yml \
-            --workspace ${{ runner.temp }}/kmd-ws \
-            --prefix ${{ runner.temp }}/prefix \
-            --release ci \
+        kmd ci/${RELEASE} ci/repository.yml --python ${{ matrix.target-python-bin }} --workspace ${{ runner.temp }}/kmd-ws --prefix ${{ runner.temp }}/prefix --release ci
 
     - name: Build libkmd.so (a dummy C library) for testing
       run: |
@@ -72,10 +66,10 @@ jobs:
     - name: Test enable script
       run: |
         source ${{ runner.temp }}/prefix/ci/enable
-        [[ "$(which python)" == "${{ runner.temp }}/prefix/ci/root/bin/python" ]]
+        [[ "${{ matrix.target-python-bin }}" == "${{ runner.temp }}/prefix/ci/root/bin/python" ]]
 
     - name: Test Python version
       run: |
         source ${{ runner.temp }}/prefix/ci/enable
-        pyver=$(python -c 'from sys import version_info as v;print(f"{v[0]}.{v[1]}")')
+        pyver=$(${{ matrix.target-python-bin }} -c 'from sys import version_info as v;print(f"{v[0]}.{v[1]}")')
         [[ "${pyver}" == "${{ matrix.python-version }}" ]]

ci/release-py3x.yml

@@ -1,4 +1,4 @@
 numpy: 1.23.5
-python: 3-builtin
+python: 3.8-venv
 setuptools: 65.6.3
 wheel: 0.38.4

ci/repository.yml

@@ -8,11 +8,10 @@ numpy:
       - python
 
 python:
-  3-builtin:
+  3.8-venv:
     make: sh
-    makefile: build__python-virtualenv.sh
+    makefile: build__python3.8-venv.sh
     maintainer: ci
-    makeopts: --virtualenv-interpreter python3
 
 setuptools:
   65.6.3:

docs/komodo.rst

@@ -205,14 +205,6 @@ komodo.snyk\_reporting module
    :undoc-members:
    :show-inheritance:
 
-komodo.switch module
---------------------
-
-.. automodule:: komodo.switch
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 komodo.yaml\_file\_type module
 ------------------------------
 

examples/repository.yml

@@ -1,9 +1,8 @@
 python:
   3-builtin:
     make: sh
-    makefile: build__python-virtualenv.sh
+    makefile: build__python3.8-venv.sh
     maintainer: [email protected]
-    makeopts: --virtualenv-interpreter python3
 
 treelib:
   1.6.1:

komodo/build.py

@@ -21,16 +21,22 @@
 flatten = itr.chain.from_iterable
 
 
+def python_bin(fakeroot: str, prefix: str) -> str:
+    """Return the path to Python as installed in the komodo release"""
+    return f"{fakeroot}/{prefix}/bin/python"
+
+
 def dfs(pkg, version, pkgs, repo):
     # package has no more dependencies - add the package itself
     if "depends" not in repo[pkg][version]:
         return [pkg]
 
-    if not all(map(pkgs.__contains__, repo[pkg][version]["depends"])):
+    depends = set(repo[pkg][version]["depends"]) - set(pkgs)
+    if depends:
         print(
             "error: "
-            + ",".join(repo[pkg][version]["depends"])
-            + " required as dependency, is not in distribution",
+            + ",".join(depends)
+            + f" required as dependency for package '{pkg}', is not in distribution",
             file=sys.stderr,
         )
         sys.exit(1)
@@ -121,7 +127,6 @@ def sh(pkg, ver, pkgpath, data, prefix, makefile, *args, **kwargs):
         cmd.append(f"--pythonpath {kwargs['pythonpath']}")
         cmd.append(f"--path {kwargs['binpath']}")
         cmd.append(f"--pip {kwargs['pip']}")
-        cmd.append(f"--virtualenv {kwargs['virtualenv']}")
         cmd.append(f"--ld-library-path {kwargs['ld_lib_path']}")
         cmd.append(kwargs.get("makeopts"))
 
@@ -196,9 +201,9 @@ def pip_install(pkg, ver, pkgpath, data, prefix, dlprefix, *args, pip="pip", **k
         ver = latest_pypi_version(pkg)
     cmd = [
         pip,
+        "--python",
+        python_bin(kwargs["fakeroot"], prefix),
         f"install {pkg}=={strip_version(ver)}",
-        f"--root {kwargs['fakeroot']}",
-        f"--prefix {prefix}",
         "--no-index",
         "--no-deps",
         "--ignore-installed",
@@ -238,7 +243,6 @@ def make(
     jobs=1,
     cmk="cmake",
     pip="pip",
-    virtualenv=None,
     fakeroot=".",
 ):
     xs = flatten(dfs(pkg, ver, pkgs, repo) for pkg, ver in pkgs.items())
@@ -331,7 +335,6 @@ def resolve(x):
             jobs=jobs,
             cmake=cmk,
             pip=pip,
-            virtualenv=virtualenv,
             fakeroot=fakeroot,
             pythonpath=build_pythonpath,
             binpath=build_path,

komodo/cli.py

@@ -4,14 +4,12 @@
 import os
 import sys
 import uuid
-import warnings
 from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Sequence, Tuple
 
 import jinja2
 from ruamel.yaml import YAML
 
-from komodo import switch
 from komodo.build import make
 from komodo.data import Data
 from komodo.fetch import fetch
@@ -25,6 +23,12 @@
 from komodo.yaml_file_types import ReleaseFile, RepositoryFile
 
 
+def _locate_cmake() -> str:
+    if os.path.isfile("/usr/bin/cmake3"):
+        return "/usr/bin/cmake3"
+    return "cmake"
+
+
 def create_enable_scripts(komodo_prefix: str, komodo_release: str) -> None:
     """Render enable scripts (user facing) for bash and csh to an existing
     directory komodo_release (in current working directory).
@@ -74,7 +78,6 @@ def _main(args):
             args.pkgs.content,
             args.repo.content,
             outdir=args.downloads,
-            pip=args.pip,
         )
         timings.append(("Fetching all packages", datetime.datetime.now() - start_time))
         _print_timings(timings[-1])
@@ -95,10 +98,6 @@ def _main(args):
             prefix=str(tmp_prefix),
             dlprefix=args.downloads,
             builddir=args.tmp,
-            jobs=args.jobs,
-            cmk=args.cmake,
-            pip=args.pip,
-            virtualenv=args.virtualenv,
             fakeroot=str(fakeroot),
         )
         timings.append(
@@ -142,7 +141,7 @@ def _main(args):
 
     start_time = datetime.datetime.now()
     shell(f"mv {args.release} .{args.release}")
-    shell(f"rsync -a .{args.release} {args.prefix}", sudo=args.sudo)
+    shell(f"rsync -a .{args.release} {args.prefix}")
     timings.append(
         (
             "Rsyncing partial komodo to destination",
@@ -155,17 +154,12 @@ def _main(args):
         shell(
             f"mv {args.prefix}/{args.release} "
             f"{args.prefix}/{args.release}.delete-{uuid.uuid4()}",
-            sudo=args.sudo,
         )
 
-    shell(
-        f"mv {args.prefix}/.{args.release} {args.prefix}/{args.release}",
-        sudo=args.sudo,
-    )
+    shell(f"mv {args.prefix}/.{args.release} {args.prefix}/{args.release}")
     start_time = datetime.datetime.now()
     shell(
         f"rm -rf {args.prefix}/{args.release}.delete-*",
-        sudo=args.sudo,
         allow_failure=True,
     )
     timings.append(("Deleting previous release", datetime.datetime.now() - start_time))
@@ -180,38 +174,10 @@ def _main(args):
     release_path = Path(args.prefix) / Path(args.release)
     release_root = release_path / "root"
     start_time = datetime.datetime.now()
-    for pkg, ver in args.pkgs.content.items():
-        current = args.repo.content[pkg][ver]
-        if current["make"] != "pip":
-            continue
-
-        package_name = current.get("pypi_package_name", pkg)
-        if ver == LATEST_PACKAGE_ALIAS:
-            ver = latest_pypi_version(package_name)
-        shell_input = [
-            args.pip,
-            f"install {package_name}=={strip_version(ver)}",
-            "--prefix",
-            str(release_root),
-            "--no-index",
-            "--no-deps",
-            "--ignore-installed",
-            # assuming fetch.py has done "pip download" to this directory:
-            f"--cache-dir {args.downloads}",
-            f"--find-links {args.downloads}",
-        ]
-        shell_input.append(current.get("makeopts"))
-
-        print(shell(shell_input, sudo=args.sudo))
-    timings.append(
-        ("pip install to final destination", datetime.datetime.now() - start_time),
-    )
     _print_timings(timings[-1])
 
     fixup_python_shebangs(args.prefix, args.release)
 
-    switch.create_activator_switch(data, args.prefix, args.release)
-
     if args.postinst:
         start_time = datetime.datetime.now()
         shell([args.postinst, release_path])
@@ -236,18 +202,20 @@ def _main(args):
         _print_timings(timing_element, adjust=True)
 
 
-def cli_main():
+def cli_main(argv: Optional[Sequence[str]] = None):
     """Pass the command-line args to argparse, then set up the workspace."""
-    args = parse_args(sys.argv[1:])
+    args = parse_args(sys.argv[1:] if argv is None else argv)
 
     if args.workspace and not Path(args.workspace).exists():
         Path(args.workspace).mkdir()
 
+    os.environ["python"] = args.python
+
     with pushd(args.workspace):
         _main(args)
 
 
-def parse_args(args: List[str]) -> argparse.Namespace:
+def parse_args(argv: Sequence[str]) -> argparse.Namespace:
     """Parse the arguments from the command line into an `argparse.Namespace`.
     Having a separated function makes it easier to test the CLI.
 
@@ -307,6 +275,12 @@ def parse_args(args: List[str]) -> argparse.Namespace:
             "containing the enable script and environment `root` directory."
         ),
     )
+    required_args.add_argument(
+        "--python",
+        type=str,
+        required=True,
+        help="Python version to build the komodo release for",
+    )
 
     optional_args = parser.add_argument_group("optional arguments")
 
@@ -338,13 +312,6 @@ def parse_args(args: List[str]) -> argparse.Namespace:
             "unless you are running with the --build option."
         ),
     )
-    optional_args.add_argument(
-        "--jobs",
-        "-j",
-        type=int,
-        default=1,
-        help="The number of parallel jobs to use for builds by cmake.",
-    )
     optional_args.add_argument(
         "--download",
         "-d",
@@ -381,34 +348,9 @@ def parse_args(args: List[str]) -> argparse.Namespace:
     optional_args.add_argument(
         "--cmake",
         type=str,
-        default="cmake",
+        default=_locate_cmake(),
         help="The command to use for cmake builds.",
     )
-    optional_args.add_argument(
-        "--pip",
-        type=str,
-        default="pip",
-        help="The command to use for pip builds.",
-    )
-    optional_args.add_argument(
-        "--virtualenv",
-        type=str,
-        default="virtualenv",
-        help="The command to use for virtual environment construction.",
-    )
-    optional_args.add_argument(
-        "--pyver",
-        type=str,
-        help="[DEPRECATED] This argument is not used.",  # Message to stderr below.
-    )
-    optional_args.add_argument(
-        "--sudo",
-        action="store_true",
-        help=(
-            "Flag to choose whether to use `sudo` for shell commands when "
-            "installing the environment."
-        ),
-    )
     optional_args.add_argument(
         "--workspace",
         type=str,
@@ -438,14 +380,7 @@ def parse_args(args: List[str]) -> argparse.Namespace:
         ),
     )
 
-    args = parser.parse_args(args)
-
-    if args.pyver is not None:
-        message = (
-            "\n\n⚠️  The --pyver option is deprecated and will be removed in a "
-            "future version of komodo. It is not used by komodo.\n"
-        )
-        warnings.warn(message, FutureWarning, stacklevel=2)
+    args = parser.parse_args(argv)
 
     return args