diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cb504c8b..b869f873c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,13 +7,13 @@ jobs: env: RELEASE: release-py3x.yml - strategy: fail-fast: false matrix: python-version: ['3.8', '3.10', '3.11'] + target-python-bin: ['/usr/bin/python3.8'] - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -30,27 +30,21 @@ jobs: sudo apt update sudo apt install csh - - name: Setup Komodo + - name: Install Komodo run: | - ./bootstrap.sh $(which python) + pip install .[dev] - name: Unit tests run: | - source boot/build-env/bin/activate - boot/kmd-env/bin/python -m pytest tests + pytest tests - name: Lint examples run: | - boot/kmd-env/bin/python -m komodo.lint examples/stable.yml examples/repository.yml + python -m komodo.lint examples/stable.yml examples/repository.yml - name: Full integration test run: | - ./run_kmd.sh \ - ci/${RELEASE} \ - ci/repository.yml \ - --workspace ${{ runner.temp }}/kmd-ws \ - --prefix ${{ runner.temp }}/prefix \ - --release ci \ + kmd ci/${RELEASE} ci/repository.yml --python ${{ matrix.target-python-bin }} --workspace ${{ runner.temp }}/kmd-ws --prefix ${{ runner.temp }}/prefix --release ci - name: Build libkmd.so (a dummy C library) for testing run: | @@ -72,10 +66,10 @@ jobs: - name: Test enable script run: | source ${{ runner.temp }}/prefix/ci/enable - [[ "$(which python)" == "${{ runner.temp }}/prefix/ci/root/bin/python" ]] + [[ "${{ matrix.target-python-bin }}" == "${{ runner.temp }}/prefix/ci/root/bin/python" ]] - name: Test Python version run: | source ${{ runner.temp }}/prefix/ci/enable - pyver=$(python -c 'from sys import version_info as v;print(f"{v[0]}.{v[1]}")') + pyver=$(${{ matrix.target-python-bin }} -c 'from sys import version_info as v;print(f"{v[0]}.{v[1]}")') [[ "${pyver}" == "${{ matrix.python-version }}" ]] diff --git a/ci/release-py3x.yml b/ci/release-py3x.yml index 5fa09c6b8..3044646bc 100644 --- a/ci/release-py3x.yml +++ b/ci/release-py3x.yml @@ -1,4 +1,4 @@ numpy: 1.23.5 -python: 3-builtin +python: 3.8-venv setuptools: 65.6.3 wheel: 0.38.4 diff --git a/ci/repository.yml b/ci/repository.yml index 353b87829..fc0f591db 100644 --- a/ci/repository.yml +++ b/ci/repository.yml @@ -8,11 +8,10 @@ numpy: - python python: - 3-builtin: + 3.8-venv: make: sh - makefile: build__python-virtualenv.sh + makefile: build__python3.8-venv.sh maintainer: ci - makeopts: --virtualenv-interpreter python3 setuptools: 65.6.3: diff --git a/docs/komodo.rst b/docs/komodo.rst index 56100eabb..5f31509a3 100644 --- a/docs/komodo.rst +++ b/docs/komodo.rst @@ -205,14 +205,6 @@ komodo.snyk\_reporting module :undoc-members: :show-inheritance: -komodo.switch module --------------------- - -.. automodule:: komodo.switch - :members: - :undoc-members: - :show-inheritance: - komodo.yaml\_file\_type module ------------------------------ diff --git a/examples/repository.yml b/examples/repository.yml index 188e7973a..fc7c66171 100644 --- a/examples/repository.yml +++ b/examples/repository.yml @@ -1,9 +1,8 @@ python: 3-builtin: make: sh - makefile: build__python-virtualenv.sh + makefile: build__python3.8-venv.sh maintainer: foo@example.com - makeopts: --virtualenv-interpreter python3 treelib: 1.6.1: diff --git a/komodo/build.py b/komodo/build.py index 290a337f9..80d48c23b 100644 --- a/komodo/build.py +++ b/komodo/build.py @@ -21,16 +21,22 @@ flatten = itr.chain.from_iterable +def python_bin(fakeroot: str, prefix: str) -> str: + """Return the path to Python as installed in the komodo release""" + return f"{fakeroot}/{prefix}/bin/python" + + def dfs(pkg, version, pkgs, repo): # package has no more dependencies - add the package itself if "depends" not in repo[pkg][version]: return [pkg] - if not all(map(pkgs.__contains__, repo[pkg][version]["depends"])): + depends = set(repo[pkg][version]["depends"]) - set(pkgs) + if depends: print( "error: " - + ",".join(repo[pkg][version]["depends"]) - + " required as dependency, is not in distribution", + + ",".join(depends) + + f" required as dependency for package '{pkg}', is not in distribution", file=sys.stderr, ) sys.exit(1) @@ -121,7 +127,6 @@ def sh(pkg, ver, pkgpath, data, prefix, makefile, *args, **kwargs): cmd.append(f"--pythonpath {kwargs['pythonpath']}") cmd.append(f"--path {kwargs['binpath']}") cmd.append(f"--pip {kwargs['pip']}") - cmd.append(f"--virtualenv {kwargs['virtualenv']}") cmd.append(f"--ld-library-path {kwargs['ld_lib_path']}") cmd.append(kwargs.get("makeopts")) @@ -196,9 +201,9 @@ def pip_install(pkg, ver, pkgpath, data, prefix, dlprefix, *args, pip="pip", **k ver = latest_pypi_version(pkg) cmd = [ pip, + "--python", + python_bin(kwargs["fakeroot"], prefix), f"install {pkg}=={strip_version(ver)}", - f"--root {kwargs['fakeroot']}", - f"--prefix {prefix}", "--no-index", "--no-deps", "--ignore-installed", @@ -238,7 +243,6 @@ def make( jobs=1, cmk="cmake", pip="pip", - virtualenv=None, fakeroot=".", ): xs = flatten(dfs(pkg, ver, pkgs, repo) for pkg, ver in pkgs.items()) @@ -331,7 +335,6 @@ def resolve(x): jobs=jobs, cmake=cmk, pip=pip, - virtualenv=virtualenv, fakeroot=fakeroot, pythonpath=build_pythonpath, binpath=build_path, diff --git a/komodo/cli.py b/komodo/cli.py index a427c6a02..5183ae4c8 100755 --- a/komodo/cli.py +++ b/komodo/cli.py @@ -4,14 +4,12 @@ import os import sys import uuid -import warnings from pathlib import Path -from typing import List, Tuple +from typing import List, Optional, Sequence, Tuple import jinja2 from ruamel.yaml import YAML -from komodo import switch from komodo.build import make from komodo.data import Data from komodo.fetch import fetch @@ -25,6 +23,12 @@ from komodo.yaml_file_types import ReleaseFile, RepositoryFile +def _locate_cmake() -> str: + if os.path.isfile("/usr/bin/cmake3"): + return "/usr/bin/cmake3" + return "cmake" + + def create_enable_scripts(komodo_prefix: str, komodo_release: str) -> None: """Render enable scripts (user facing) for bash and csh to an existing directory komodo_release (in current working directory). @@ -74,7 +78,6 @@ def _main(args): args.pkgs.content, args.repo.content, outdir=args.downloads, - pip=args.pip, ) timings.append(("Fetching all packages", datetime.datetime.now() - start_time)) _print_timings(timings[-1]) @@ -95,10 +98,6 @@ def _main(args): prefix=str(tmp_prefix), dlprefix=args.downloads, builddir=args.tmp, - jobs=args.jobs, - cmk=args.cmake, - pip=args.pip, - virtualenv=args.virtualenv, fakeroot=str(fakeroot), ) timings.append( @@ -142,7 +141,7 @@ def _main(args): start_time = datetime.datetime.now() shell(f"mv {args.release} .{args.release}") - shell(f"rsync -a .{args.release} {args.prefix}", sudo=args.sudo) + shell(f"rsync -a .{args.release} {args.prefix}") timings.append( ( "Rsyncing partial komodo to destination", @@ -155,17 +154,12 @@ def _main(args): shell( f"mv {args.prefix}/{args.release} " f"{args.prefix}/{args.release}.delete-{uuid.uuid4()}", - sudo=args.sudo, ) - shell( - f"mv {args.prefix}/.{args.release} {args.prefix}/{args.release}", - sudo=args.sudo, - ) + shell(f"mv {args.prefix}/.{args.release} {args.prefix}/{args.release}") start_time = datetime.datetime.now() shell( f"rm -rf {args.prefix}/{args.release}.delete-*", - sudo=args.sudo, allow_failure=True, ) timings.append(("Deleting previous release", datetime.datetime.now() - start_time)) @@ -180,38 +174,10 @@ def _main(args): release_path = Path(args.prefix) / Path(args.release) release_root = release_path / "root" start_time = datetime.datetime.now() - for pkg, ver in args.pkgs.content.items(): - current = args.repo.content[pkg][ver] - if current["make"] != "pip": - continue - - package_name = current.get("pypi_package_name", pkg) - if ver == LATEST_PACKAGE_ALIAS: - ver = latest_pypi_version(package_name) - shell_input = [ - args.pip, - f"install {package_name}=={strip_version(ver)}", - "--prefix", - str(release_root), - "--no-index", - "--no-deps", - "--ignore-installed", - # assuming fetch.py has done "pip download" to this directory: - f"--cache-dir {args.downloads}", - f"--find-links {args.downloads}", - ] - shell_input.append(current.get("makeopts")) - - print(shell(shell_input, sudo=args.sudo)) - timings.append( - ("pip install to final destination", datetime.datetime.now() - start_time), - ) _print_timings(timings[-1]) fixup_python_shebangs(args.prefix, args.release) - switch.create_activator_switch(data, args.prefix, args.release) - if args.postinst: start_time = datetime.datetime.now() shell([args.postinst, release_path]) @@ -236,18 +202,20 @@ def _main(args): _print_timings(timing_element, adjust=True) -def cli_main(): +def cli_main(argv: Optional[Sequence[str]] = None): """Pass the command-line args to argparse, then set up the workspace.""" - args = parse_args(sys.argv[1:]) + args = parse_args(sys.argv[1:] if argv is None else argv) if args.workspace and not Path(args.workspace).exists(): Path(args.workspace).mkdir() + os.environ["python"] = args.python + with pushd(args.workspace): _main(args) -def parse_args(args: List[str]) -> argparse.Namespace: +def parse_args(argv: Sequence[str]) -> argparse.Namespace: """Parse the arguments from the command line into an `argparse.Namespace`. Having a separated function makes it easier to test the CLI. @@ -307,6 +275,12 @@ def parse_args(args: List[str]) -> argparse.Namespace: "containing the enable script and environment `root` directory." ), ) + required_args.add_argument( + "--python", + type=str, + required=True, + help="Python version to build the komodo release for", + ) optional_args = parser.add_argument_group("optional arguments") @@ -338,13 +312,6 @@ def parse_args(args: List[str]) -> argparse.Namespace: "unless you are running with the --build option." ), ) - optional_args.add_argument( - "--jobs", - "-j", - type=int, - default=1, - help="The number of parallel jobs to use for builds by cmake.", - ) optional_args.add_argument( "--download", "-d", @@ -381,34 +348,9 @@ def parse_args(args: List[str]) -> argparse.Namespace: optional_args.add_argument( "--cmake", type=str, - default="cmake", + default=_locate_cmake(), help="The command to use for cmake builds.", ) - optional_args.add_argument( - "--pip", - type=str, - default="pip", - help="The command to use for pip builds.", - ) - optional_args.add_argument( - "--virtualenv", - type=str, - default="virtualenv", - help="The command to use for virtual environment construction.", - ) - optional_args.add_argument( - "--pyver", - type=str, - help="[DEPRECATED] This argument is not used.", # Message to stderr below. - ) - optional_args.add_argument( - "--sudo", - action="store_true", - help=( - "Flag to choose whether to use `sudo` for shell commands when " - "installing the environment." - ), - ) optional_args.add_argument( "--workspace", type=str, @@ -438,14 +380,7 @@ def parse_args(args: List[str]) -> argparse.Namespace: ), ) - args = parser.parse_args(args) - - if args.pyver is not None: - message = ( - "\n\n⚠️ The --pyver option is deprecated and will be removed in a " - "future version of komodo. It is not used by komodo.\n" - ) - warnings.warn(message, FutureWarning, stacklevel=2) + args = parser.parse_args(argv) return args diff --git a/komodo/data/activator_switch.csh.tmpl b/komodo/data/activator_switch.csh.tmpl deleted file mode 100644 index 643b291a4..000000000 --- a/komodo/data/activator_switch.csh.tmpl +++ /dev/null @@ -1,12 +0,0 @@ -if ( `uname -r` =~ *el7* ) then - setenv KOMODO_ROOT {{ prefix }} - set KOMODO_RELEASE_REAL = "{{ release }}" - - source $KOMODO_ROOT/$KOMODO_RELEASE_REAL-rhel7/enable.csh - if ( $?_KOMODO_OLD_PROMPT ) then - set prompt = "[$KOMODO_RELEASE_REAL] $_KOMODO_OLD_PROMPT" - endif - setenv KOMODO_RELEASE $KOMODO_RELEASE_REAL -else - echo -e "{{ migration_warning }}" -endif diff --git a/komodo/data/activator_switch.tmpl b/komodo/data/activator_switch.tmpl deleted file mode 100644 index a287d61d1..000000000 --- a/komodo/data/activator_switch.tmpl +++ /dev/null @@ -1,10 +0,0 @@ -if [[ $(uname -r) == *el7* ]] ; then - export KOMODO_ROOT={{ prefix }} - KOMODO_RELEASE_REAL={{ release }} - - source $KOMODO_ROOT/$KOMODO_RELEASE_REAL-rhel7/enable - export PS1="(${KOMODO_RELEASE_REAL}) ${_PRE_KOMODO_PS1}" - export KOMODO_RELEASE=$KOMODO_RELEASE_REAL -else - echo -e "{{ migration_warning }}" -fi diff --git a/komodo/data/build__python3.8-venv.sh b/komodo/data/build__python3.8-venv.sh new file mode 100644 index 000000000..9dd93dfac --- /dev/null +++ b/komodo/data/build__python3.8-venv.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -xe + +while test $# -gt 0; do + case "$1" in + --fakeroot) + shift + export FAKEROOT=$1 + ;; + --prefix) + shift + export PREFIX=$1 + ;; + --virtualenv) + shift + export VIRTUALENV=$1 + ;; + --virtualenv-interpreter) + shift + export VIRTUALENV_INTERPRETER=$1 + ;; + *) + export OPTS="$OPTS $1" + ;; + esac + shift +done +out=${FAKEROOT}${PREFIX} + +# Create venv +$python -m venv --symlinks --without-pip $out >&2 + +# Delete unwanted scripts +rm -f $out/bin/activate* +rm -f $out/bin/Activate.ps1 + +# Replace binaries with shims +for bin in $out/bin/python*; do + rm $bin + cat >$bin <&2 +echo " /opt/rh/rh-python38/root/usr/bin/python3.8" >&2 +echo " /usr/bin/python3.8" >&2 +echo " $python" >&2 +exit 1 +EOF + chmod +x $bin +done diff --git a/komodo/fetch.py b/komodo/fetch.py index a3a628f02..e7528b12d 100644 --- a/komodo/fetch.py +++ b/komodo/fetch.py @@ -29,6 +29,8 @@ def grab(path, filename=None, version=None, protocol=None, pip="pip"): if protocol in ("http", "https", "ftp"): shell(f"wget --quiet {path} -O {filename}") elif protocol in ("git"): + if os.path.exists(filename): + return shell( "git clone " f"-b {strip_version(version)} " @@ -57,6 +59,7 @@ def grab(path, filename=None, version=None, protocol=None, pip="pip"): def fetch(pkgs, repo, outdir, pip="pip") -> dict: + python = os.environ["python"] missingpkg = [pkg for pkg in pkgs if pkg not in repo] missingver = [ pkg for pkg, ver in pkgs.items() if pkg in repo and ver not in repo[pkg] @@ -80,11 +83,6 @@ def fetch(pkgs, repo, outdir, pip="pip") -> dict: raise ValueError( msg, ) - if os.path.exists(outdir) and os.listdir(outdir): - msg = f"Downloading to non-empty directory {outdir} is not supported." - raise RuntimeError( - msg, - ) if not os.path.exists(outdir): os.mkdir(outdir) @@ -123,7 +121,7 @@ def fetch(pkgs, repo, outdir, pip="pip") -> dict: f"Nothing to fetch for {pkgname}, " f"but created folder {package_folder}", ) - os.mkdir(pkgname) + os.makedirs(pkgname, exist_ok=True) continue dst = pkgname @@ -158,7 +156,17 @@ def fetch(pkgs, repo, outdir, pip="pip") -> dict: os.symlink(normalised_dir, pkgname) print(f"Downloading {len(pypi_packages)} pypi packages") - shell([pip, "download", "--no-deps", "--dest .", " ".join(pypi_packages)]) + shell( + [ + pip, + "--python", + python, + "download", + "--no-deps", + "--dest .", + " ".join(pypi_packages), + ] + ) return git_hashes @@ -199,5 +207,4 @@ def fetch(pkgs, repo, outdir, pip="pip") -> dict: args.content.pkgfile, args.content.repofile, outdir=args.content.output, - pip=args.pip, ) diff --git a/komodo/matrix.py b/komodo/matrix.py index 0ed1977f9..f30798323 100644 --- a/komodo/matrix.py +++ b/komodo/matrix.py @@ -3,28 +3,25 @@ handling an arbitrary large and funky matrix, without having to guess and/or repeat itself. """ -import itertools import re -from typing import Iterator, Sequence, Tuple +from typing import Iterable, Sequence def get_matrix( - rhel_versions: Sequence[str], py_versions: Sequence[str], -) -> Iterator[Tuple[str, str]]: +) -> Iterable[str]: """Return tuples of rhel version and Python version, representing the current release matrix. """ - for product in itertools.product(rhel_versions, py_versions): - rh_ver, py_ver = product - yield (f"rhel{rh_ver}", f"py{str(py_ver).replace('.', '')}") + for py_ver in py_versions: + yield f"py{str(py_ver).replace('.', '')}" -def format_release(base: str, rhel_ver: str, py_ver: str) -> str: +def format_release(base: str, py_ver: str) -> str: """Format a base (e.g. a matrix file without the .yml suffix) such that it looks like a concrete release. """ - return f"{base}-{py_ver}-{rhel_ver}" + return f"{base}-{py_ver}" def get_matrix_base(release_name: str) -> str: @@ -32,7 +29,7 @@ def get_matrix_base(release_name: str) -> str: inverse of format_release for actual, concrete matrix releases. Hard-coded the suffix pattern '-py..-rhel.' or '-py...-rhel.'. """ - suffix = format_release("", "rhel[0-9]", r"py\d{2,3}") + suffix = format_release("", r"py\d{2,3}") if re.search(suffix, release_name): return re.split(suffix, release_name)[0] # no matrix suffix at all diff --git a/komodo/release_transpiler.py b/komodo/release_transpiler.py deleted file mode 100755 index 1c2addb84..000000000 --- a/komodo/release_transpiler.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python - -import argparse -import itertools -import os -from typing import Dict, List, Optional, Sequence - -import yaml -from packaging import version as version_parser - -from komodo.matrix import format_release, get_matrix -from komodo.prettier import load_yaml, write_to_file - - -def build_matrix_file( - release_base: str, - release_folder: str, - builtins: dict, - py_coords: Optional[Sequence[str]], -) -> None: - """Combine release files from the release_folder into one single matrix_file.""" - files = {} - if py_coords is None: - py_keys = get_py_coords(release_base, release_folder) - else: - py_keys = [f"py{py_version.replace('.', '')}" for py_version in py_coords] - - for key in py_keys: - files[key] = load_yaml(f"{release_folder}/{release_base}-{key}.yml") - - all_packages = set( - itertools.chain.from_iterable(files[key].keys() for key in files), - ) - compiled = {} - - for package in all_packages: - if package in builtins: - compiled[package] = builtins[package] - continue - - if len({files[key].get(package) for key in files}) == 1: - compiled[package] = next(iter(files.values()))[package] - else: - compiled[package] = {key: files[key].get(package) for key in py_keys} - - write_to_file(compiled, f"{release_base}.yml", False) - - -def get_py_coords(release_base: str, release_folder: str) -> Sequence[str]: - """Get python versions of release files inside a given release_folder.""" - filenames_with_prefix = sorted( - [ - filename - for filename in os.listdir(release_folder) - if filename.startswith(release_base) - ], - ) - len_release_base = len(release_base + "-") - irrelevant_suffix_length = len(".yml") - return [ - filename[len_release_base:-irrelevant_suffix_length] - for filename in filenames_with_prefix - ] - - -def _pick_package_versions_for_release( - packages: dict, - rhel_ver: str, - py_ver: str, -) -> dict: - """Consolidate the packages for a given combination of rhel and python version - into a dictionary. - """ - release_dict = {} - for pkg_name, versions in packages.items(): - try: - _check_version_exists_for_coordinates(versions, rhel_ver, py_ver) - except KeyError as err: - error_msg = f"{err!s}. Failed for {pkg_name}." - raise KeyError(error_msg) from None - - if rhel_ver in versions: - version = versions[rhel_ver][py_ver] - elif py_ver in versions: - version = versions[py_ver] - else: - version = versions - - if version: - release_dict[pkg_name] = version - return release_dict - - -def _check_version_exists_for_coordinates( - pkg_versions: dict, - rhel_coordinate: str, - py_coordinate: str, -) -> None: - """Check the coordinates `rhel_ver` and `py_ver` input as arguments to - build a release against the release matrix file. Raise exceptions if - coordinates not found. - pkg_versions can take various levels as the examples show: - { - rhel7: # first_level - py36: 1.1.1, # second_level - py38: 2.1.1, - rhel8: # first_level - py36: 3.1.1, # second_level - py38: 4.1.1, - } - or: - { - py36: 1.1.1, # first level - py38: 2.1.1, - } - or: - {1.1.1}. - - """ - first_level_versions = [] - for version in pkg_versions: - first_level_versions.append(version) - if "rhel" in first_level_versions[0]: - # Both rhel and python versions can have different versions - if rhel_coordinate not in first_level_versions: - msg = f"Rhel version {rhel_coordinate} not found." - raise KeyError(msg) - second_level_versions = [] - for version_py in pkg_versions[rhel_coordinate]: - second_level_versions.append(version_py) - if py_coordinate not in second_level_versions: - msg = f"Python version {py_coordinate} not found for rhel version {rhel_coordinate}." - raise KeyError( - msg, - ) - elif "py" in first_level_versions[0] and py_coordinate not in first_level_versions: - # Only python has different versions - msg = f"Python version {py_coordinate} not found." - raise KeyError(msg) - - -def transpile_releases(matrix_file: str, output_folder: str, matrix: dict) -> None: - """Transpile a matrix file possibly containing different os and framework - versions (e.g. rhel6 and rhel7, py3.6 and py3.8). - Write one dimension file for each element in the matrix - (e.g. rhel7 and py3.8, rhel6 and py3.6). - """ - rhel_versions = matrix["rhel"] - python_versions = matrix["py"] - - release_base = os.path.splitext(os.path.basename(matrix_file))[0] - release_folder = os.path.dirname(matrix_file) - release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") - for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): - release_dict = _pick_package_versions_for_release( - release_matrix, - rhel_ver, - py_ver, - ) - filename = f"{format_release(release_base, rhel_ver, py_ver)}.yml" - write_to_file(release_dict, os.path.join(output_folder, filename)) - - -def transpile_releases_for_pip( - matrix_file: str, - output_folder: str, - repository_file: str, - matrix: dict, -) -> None: - rhel_versions = matrix["rhel"] - python_versions = matrix["py"] - release_base = os.path.splitext(os.path.basename(matrix_file))[0] - release_folder = os.path.dirname(matrix_file) - release_matrix = load_yaml(f"{os.path.join(release_folder, release_base)}.yml") - repository = load_yaml(repository_file) - for rhel_ver, py_ver in get_matrix(rhel_versions, python_versions): - release_dict = _pick_package_versions_for_release( - release_matrix, - rhel_ver, - py_ver, - ) - pip_packages = [ - f"{pkg}=={version}" - for pkg, version in release_dict.items() - if repository[pkg][version].get("make") == "pip" - ] - filename = f"{format_release(release_base, rhel_ver, py_ver)}.req" - with open( - os.path.join(output_folder, filename), - mode="w", - encoding="utf-8", - ) as filehandler: - filehandler.write("\n".join(pip_packages)) - - -def combine(args): - build_matrix_file( - args.release_base, - args.release_folder, - load_yaml(args.override_mapping), - args.py_coords, - ) - - -def transpile(args): - transpile_releases(args.matrix_file, args.output_folder, args.matrix_coordinates) - - -def transpile_for_pip(args: Dict): - transpile_releases_for_pip( - args.matrix_file, - args.output_folder, - args.repo, - args.matrix_coordinates, - ) - - -def main(): - parser = argparse.ArgumentParser( - description="Build release files.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - subparsers = parser.add_subparsers( - title="Commands", - description="Combine - build matrix file\nTranspile - generate release files", - help="Available sub commands", - dest="mode", - ) - subparsers.required = True - - matrix_parser = subparsers.add_parser( - "combine", - description=""" -Combine release files into a matrix file. Output format: - - example-package: - rhel7: - py36 : # package not included in release - py38 : 5.11.13 - rhel8: - py36 : # package not included in release - py38 : 5.11.13+builtin""", - formatter_class=argparse.RawTextHelpFormatter, - ) - matrix_parser.set_defaults(func=combine) - matrix_parser.add_argument( - "--release-base", - required=True, - help="Name of the release to handle (default: None)", - ) - matrix_parser.add_argument( - "--release-folder", - required=True, - type=dir_path, - help="Folder with existing release file (default: None)", - ) - matrix_parser.add_argument( - "--override-mapping", - required=True, - type=valid_file, - help="File containing explicit matrix packages (default: None)", - ) - - def comma_delimited_python_versions(python_versions: str) -> List[str]: - output_list: List[str] = [] - for python_version in python_versions.split(","): - parsed_python_version = version_parser.parse(python_version).base_version - if parsed_python_version >= "4.0.0": - raise version_parser.InvalidVersion(python_version) - output_list.append(parsed_python_version) - return output_list - - matrix_parser.add_argument( - "--py_coords", - help="""Comma delimited list of python versions to be combined, - for example, "3.6,3.8" (without spaces). - If None, the release files in release-folder will be used to imply - the versions to combine.""", - type=comma_delimited_python_versions, - required=False, - default=None, - ) - - transpile_parser = subparsers.add_parser( - "transpile", - description="Transpile a matrix file into separate release files.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - transpile_parser.set_defaults(func=transpile) - transpile_parser.add_argument( - "--matrix-file", - required=True, - type=valid_file, - help="Yaml file describing the release matrix", - ) - transpile_parser.add_argument( - "--output-folder", - required=True, - type=dir_path, - help="Folder to output new release files", - ) - transpile_parser.add_argument( - "--matrix-coordinates", - help="Matrix to be transpiled, expected yaml format string.", - type=yaml.safe_load, - required=False, - default="{rhel: ['7'], py: ['3.8']}", - ) - transpile_for_pip_parser = subparsers.add_parser( - "transpile-for-pip", - description="transpile a matrix file into separate pip requirement files.", - ) - transpile_for_pip_parser.set_defaults(func=transpile_for_pip) - transpile_for_pip_parser.add_argument( - "--matrix-file", - required=True, - help="Yaml file describing the release matrix", - ) - transpile_for_pip_parser.add_argument( - "--repo", - required=True, - help="A Komodo repository file, in YAML format.", - ) - transpile_for_pip_parser.add_argument( - "--output-folder", - required=True, - help="Folder to output new release files", - ) - transpile_for_pip_parser.add_argument( - "--matrix-coordinates", - help="Matrix to be transpiled, expected yaml format string.", - type=yaml.safe_load, - required=False, - default="{rhel: ['7'], py: ['3.8']}", - ) - args = parser.parse_args() - args.func(args) - - -def valid_file(path: str) -> str: - if os.path.isfile(path): - return path - raise FileNotFoundError(path) - - -def dir_path(should_be_valid_path: str) -> str: - if os.path.isdir(should_be_valid_path): - return should_be_valid_path - raise NotADirectoryError(should_be_valid_path) - - -if __name__ == "__main__": - main() diff --git a/komodo/shell.py b/komodo/shell.py index 9b7ea3121..fbb7dbaef 100644 --- a/komodo/shell.py +++ b/komodo/shell.py @@ -32,7 +32,7 @@ def shell(cmd: str, sudo=False, allow_failure=False) -> bytes: try: return subprocess.check_output(tuple(filter(None, cmdlist))) except subprocess.CalledProcessError as e: - print(e.output, file=sys.stderr) + print(e.output.decode("utf-8", "replace"), file=sys.stderr) if allow_failure: return e.output raise diff --git a/komodo/show_version.py b/komodo/show_version.py index bcb8601e1..fc647d94c 100644 --- a/komodo/show_version.py +++ b/komodo/show_version.py @@ -84,7 +84,7 @@ def get_komodo_path(release: str) -> Path: """Use the release name to find the 'real' release path in an ordinary komodo environment. E.g., the release may be something like '2023.01.02-py38' but the 'real' release (where the release manifest - is stored) might be platform-specific, e.g. '2023.01.02-py38-rhel7'. + is stored) might be platform-specific, e.g. '2023.01.02-py38'. The real path is in the PATH, so we try to get it from there. Args: diff --git a/komodo/switch.py b/komodo/switch.py deleted file mode 100644 index 43a7f058d..000000000 --- a/komodo/switch.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -import shutil - -from jinja2 import Template - -MIGRATION_WARNING = ( - "Attention! Your machine is running on an environment " - "that is not supported. RHEL6 has been phased out.\n" - "From October 2020, komodo versions only support RHEL7.\n" - "Please migrate as soon as possible. If you have any questions or issues - " - "contact us on #ert-users on Slack or Equinor's Yammer." -) - - -def create_activator_switch(data, prefix, release): - """Given a prefix and a release, create an activator switch which - will vary the selected activator based on the RHEL version. The data - argument is expected to be a komodo.data.Data instance. - """ - # drop "-rheln" - try: - release_py, _ = release.rsplit("-", 1) - except ValueError: - # likely a build that does not require an activator switch - return - - release_path = os.path.join(prefix, release_py) - if os.path.exists(release_path): - if os.path.islink(release_path): - os.unlink(release_path) - else: - shutil.rmtree(release_path) - - os.makedirs(release_path) - - with open( - os.path.join(release_path, "enable"), "w", encoding="utf-8" - ) as activator, open( - data.get("activator_switch.tmpl"), encoding="utf-8" - ) as activator_tmpl: - activator.write( - Template(activator_tmpl.read(), keep_trailing_newline=True).render( - prefix=prefix, - release=release_py, - migration_warning=MIGRATION_WARNING, - ), - ) - - with open( - os.path.join(release_path, "enable.csh"), "w", encoding="utf-8" - ) as activator, open( - data.get("activator_switch.csh.tmpl"), encoding="utf-8" - ) as activator_tmpl: - activator.write( - Template(activator_tmpl.read(), keep_trailing_newline=True).render( - prefix=prefix, - release=release_py, - migration_warning=MIGRATION_WARNING, - ), - ) diff --git a/komodo/yaml_file_types.py b/komodo/yaml_file_types.py index 06de5d764..054353e2d 100644 --- a/komodo/yaml_file_types.py +++ b/komodo/yaml_file_types.py @@ -112,10 +112,9 @@ def lint_release_name(packagefile_path: str) -> List[komodo_error]: relname = os.path.basename(packagefile_path) found = False for py_suffix in "-py27", "-py36", "-py38", "-py310": - for rh_suffix in "", "-rhel6", "-rhel7", "-rhel8": - if relname.endswith(py_suffix + rh_suffix + ".yml"): - found = True - break + if relname.endswith(f"{py_suffix}.yml"): + found = True + break if not found: return [ _komodo_error( diff --git a/pyproject.toml b/pyproject.toml index 31797e40a..00d6149c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,6 @@ komodo-reverse-deps = "komodo.reverse_dep_graph:main" komodo-show-version = "komodo.show_version:main" komodo-snyk-test = "komodo.snyk_reporting:main" komodo-suggest-symlinks = "komodo.symlink.suggester.cli:main" -komodo-transpiler = "komodo.release_transpiler:main" [tool.pylint.messages_control] disable = [ @@ -116,6 +115,7 @@ select = [ "B", # flake-8-bugbear "SIM", # flake-8-simplify ] +ignore = ["SIM112"] # allow non-uppercase environment variables line-length = 88 [tool.ruff.isort] diff --git a/tests/conftest.py b/tests/conftest.py index 83b3654a3..8fec7aa11 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,46 +1,34 @@ -import os +import sys from unittest.mock import mock_open, patch import pytest @pytest.fixture() -def mock_komodo_env_vars(): +def mock_komodo_env_vars(monkeypatch): """Provide the environment vars from a komodo environment.""" - env = { - "PATH": "/foo/bar/komodo-release-0.0.1/root/bin", - "KOMODO_RELEASE": "komodo-release-0.0.1", - } - with patch.dict(os.environ, env): - yield + monkeypatch.setenv("PATH", "/foo/bar/komodo-release-0.0.1/root/bin") + monkeypatch.setenv("KOMODO_RELEASE", "komodo-release-0.0.1") @pytest.fixture() -def mock_bad_komodo_env_vars(): +def mock_bad_komodo_env_vars(monkeypatch): """Provide environment vars from a komodo environment, but the variables are not consistent with each other. """ - env = { - "PATH": "/foo/bar/komodo-release-99.99.99/root/bin", - "KOMODO_RELEASE": "komodo-release-0.0.1", - } - with patch.dict(os.environ, env): - yield + monkeypatch.setenv("PATH", "/foo/bar/komodo-release-99.99.99/root/bin") + monkeypatch.setenv("KOMODO_RELEASE", "komodo-release-0.0.1") @pytest.fixture() -def mock_komodoenv_env_vars(): +def mock_komodoenv_env_vars(monkeypatch): """Provide the environment vars for a *komodoenv* environment. These are different from komodo environments in that the name will not necessarily match an original komodo release, and the given path does not contain a komodo manifest file. """ - env = { - "PATH": "/quux/komodo-release/root/bin", - "KOMODO_RELEASE": "/quux/komodo-release", - } - with patch.dict(os.environ, env): - yield + monkeypatch.setenv("PATH", "/quux/komodo-release/root/bin") + monkeypatch.setenv("KOMODO_RELEASE", "/quux/komodo-release") @pytest.fixture() @@ -88,3 +76,11 @@ def captured_shell_commands(monkeypatch): m.setattr("komodo.build.shell", lambda cmd: commands.append(cmd)) m.setattr("komodo.fetch.shell", lambda cmd: commands.append(cmd)) yield commands + + +@pytest.fixture(autouse=True) +def setenv_python(monkeypatch): + import komodo.build + + monkeypatch.setenv("python", sys.executable) + monkeypatch.setattr(komodo.build, "python_bin", lambda *_: sys.executable) diff --git a/tests/test_build.py b/tests/test_build.py index 522b44f54..f53377c03 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,3 +1,4 @@ +import sys from unittest.mock import patch import pytest @@ -29,7 +30,9 @@ def test_make_one_pip_package(captured_shell_commands, tmpdir): assert len(captured_shell_commands) == 2 assert "mkdir" in " ".join(captured_shell_commands[0]) - assert "pip install pyaml" in " ".join(captured_shell_commands[1]) + assert f"pip --python {sys.executable} install pyaml" in " ".join( + captured_shell_commands[1] + ) def test_make_one_pip_package_different_name(captured_shell_commands, tmpdir): @@ -50,7 +53,9 @@ def test_make_one_pip_package_different_name(captured_shell_commands, tmpdir): assert len(captured_shell_commands) == 2 assert "mkdir" in " ".join(captured_shell_commands[0]) - assert "pip install PyYaml==20.4.0" in " ".join(captured_shell_commands[1]) + assert f"pip --python {sys.executable} install PyYaml==20.4.0" in " ".join( + captured_shell_commands[1] + ) def test_make_pip_package_from_latest(captured_shell_commands, tmpdir): @@ -73,7 +78,9 @@ def test_make_pip_package_from_latest(captured_shell_commands, tmpdir): mock_latest_version.assert_called_once_with("PyYaml") assert len(captured_shell_commands) == 2 assert "mkdir" in " ".join(captured_shell_commands[0]) - assert "pip install PyYaml==1.0.0" in " ".join(captured_shell_commands[1]) + assert f"pip --python {sys.executable} install PyYaml==1.0.0" in " ".join( + captured_shell_commands[1] + ) def test_make_sh_does_not_accept_pypi_package_name(captured_shell_commands, tmpdir): diff --git a/tests/test_cli.py b/tests/test_cli.py index 11692c46f..19f365b39 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,11 +4,16 @@ import pytest -from komodo.cli import cli_main, parse_args +from komodo.cli import cli_main from komodo.package_version import LATEST_PACKAGE_ALIAS from tests import _get_test_root +@pytest.fixture(autouse=True) +def dummy_python(monkeypatch): + monkeypatch.setattr(sys, "executable", "/usr/bin/true") + + @pytest.mark.parametrize( "args", [ @@ -24,7 +29,7 @@ "tmp", "--release", "nominal_release", - "--pip", + "--python", "/bin/true", os.path.join(_get_test_root(), "data/cli/nominal_release.yml"), os.path.join(_get_test_root(), "data/cli/nominal_repository.yml"), @@ -86,58 +91,32 @@ def test_main(args, tmpdir): assert "version: 7f4405928bd16de496522d9301c377c7bcca5ef0" in releasedoc_content -@pytest.mark.parametrize( - "args", - [ - ( +def test_minimal_main(tmp_path): + """Check that a minimal example, more like that from the README, also works. + + Without --locations-config, this should not produce the scripts local & local.csh. + """ + release_name = "minimal_release" + release_path = tmp_path / "prefix" / release_name + + cli_main( + [ + "--workspace", + str(tmp_path), os.path.join(_get_test_root(), "data/cli/minimal_release.yml"), os.path.join(_get_test_root(), "data/cli/minimal_repository.yml"), "--prefix", "prefix", "--release", - "minimal_release", + release_name, + "--python", + sys.executable, "--extra-data-dirs", # Required to find test_python_builtin.sh. os.path.join(_get_test_root(), "data/cli"), - ), - ], -) -def test_minimal_main(args, tmpdir): - """Check that a minimal example, more like that from the README, also works. - - Without --locations-config, this should not produce the scripts local & local.csh. - """ - tmpdir = str(tmpdir) - - sys.argv = [ - "kmd", - "--workspace", - tmpdir, - ] - - sys.argv.extend(list(args)) - - cli_main() - - release_name = args[5] - release_path = os.path.join(tmpdir, "prefix", release_name) - - assert os.path.exists(os.path.join(release_path, "enable")) - assert os.path.exists(os.path.join(release_path, "enable.csh")) - assert not os.path.exists(os.path.join(release_path, "local")) - assert not os.path.exists(os.path.join(release_path, "local.csh")) - - -def test_pyver_is_deprecated(capsys): - """Pyver is not being used anywhere in the code and has been deprecated. - This test ensures that its use prints a message in stderr. - - Note that one can raise a DeprecationWarning instead, and test for it, - but it does not show up in the CLI. - """ - pkgs = os.path.join(_get_test_root(), "data/cli/nominal_release.yml") - repo = os.path.join(_get_test_root(), "data/cli/nominal_repository.yml") - cmd = f"{pkgs} {repo} --prefix pfx --release rel --pyver 3.8" - with pytest.warns(FutureWarning) as record: - _ = parse_args(cmd.split()) + ] + ) - assert "The --pyver option is deprecated" in record[0].message.args[0] + assert (release_path / "enable").exists() + assert (release_path / "enable.csh").exists() + assert not (release_path / "local").exists() + assert not (release_path / "local.csh").exists() diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 76ffb8a45..936eca2bf 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,4 +1,5 @@ import os +import sys from unittest.mock import patch import pytest @@ -26,7 +27,7 @@ def test_make_one_pip_package(captured_shell_commands, tmpdir): command = " ".join(captured_shell_commands[0]) - assert command.startswith("pip download") + assert command.startswith(f"pip --python {sys.executable} download") assert "pyaml" in command @@ -46,7 +47,7 @@ def test_version_plus_marker(captured_shell_commands, tmpdir): assert len(captured_shell_commands) == 1 command = " ".join(captured_shell_commands[0]) - assert command.startswith("pip download") + assert command.startswith(f"pip --python {sys.executable} download") assert "ert==2.25.0" in command @@ -69,7 +70,7 @@ def test_allow_pre_release_with_dash(captured_shell_commands, tmpdir): command = " ".join(captured_shell_commands[0]) - assert command.startswith("pip download") + assert command.startswith(f"pip --python {sys.executable} download") assert "ert==2.25.0-rc1" in command @@ -92,7 +93,7 @@ def test_fetch_with_empty_pypi_package_name(captured_shell_commands, tmpdir): command = " ".join(captured_shell_commands[0]) - assert command.startswith("pip download") + assert command.startswith(f"pip --python {sys.executable} download") assert "PyYaml" in command diff --git a/tests/test_matrix.py b/tests/test_matrix.py index d8120fee8..b6ce7137e 100644 --- a/tests/test_matrix.py +++ b/tests/test_matrix.py @@ -4,17 +4,17 @@ def test_format_matrix(): - assert matrix.format_release("base", "rhel6", "py27") == "base-py27-rhel6" + assert matrix.format_release("base", "py27") == "base-py27" @pytest.mark.parametrize( ("test_input", "expected"), [ - ("1970.12.01-py38-rhel7", "1970.12.01"), - ("1970.12.rc0-foo-py38-rhel7", "1970.12.rc0-foo"), + ("1970.12.01-py38", "1970.12.01"), + ("1970.12.rc0-foo-py38", "1970.12.rc0-foo"), ("1970.12.03", "1970.12.03"), - (matrix.format_release("1970.12.04", "rhel7", "py38"), "1970.12.04"), - ("1970.12.05-rhel8-py27", "1970.12.05-rhel8-py27"), # outside matrix + (matrix.format_release("1970.12.04", "py38"), "1970.12.04"), + ("1970.12.05-py27", "1970.12.05"), # outside matrix ], ) def test_get_matrix_base(test_input, expected): diff --git a/tests/test_release_transpiler.py b/tests/test_release_transpiler.py deleted file mode 100644 index f394cc414..000000000 --- a/tests/test_release_transpiler.py +++ /dev/null @@ -1,129 +0,0 @@ -import os - -import pytest -import yaml - -from komodo.release_transpiler import ( - build_matrix_file, - get_py_coords, - transpile_releases, - transpile_releases_for_pip, -) -from tests import _get_test_root - -builtins = { - "lib1": { - "rhel6": {"py27": "0.1.2", "py36": "1.2.3", "py38": "1.2.4"}, - "rhel7": { - "py27": "0.1.2+builtin", - "py36": "1.2.3+builtin", - "py38": "1.2.3+builtin", - }, - }, -} - - -@pytest.mark.parametrize( - ("py_coords_input", "packages_lib2", "py_coords_not_in_lib3"), - [ - (["3.6", "3.8"], "2.3.4", "py27"), - (None, {"py27": "1.2.3", "py36": "2.3.4", "py38": "2.3.4"}, [""]), - ], -) -def test_build_release_matrix_py_coords( - tmpdir, - py_coords_input, - packages_lib2, - py_coords_not_in_lib3, -): - """lib1 tests packages with builtins, - lib2 tests packages with same version for two py coordinates, - lib3 tests packages with different versions for each py coordinate. - """ - release_base = "2020.01.a1" - release_folder = os.path.join(_get_test_root(), "data/test_releases/") - with tmpdir.as_cwd(): - build_matrix_file(release_base, release_folder, builtins, py_coords_input) - new_release_file = f"{release_base}.yml" - assert os.path.isfile(new_release_file) - with open(new_release_file, encoding="utf-8") as f: - release_matrix = yaml.safe_load(f) - - assert release_matrix["lib1"] == builtins["lib1"] - assert release_matrix["lib2"] == packages_lib2 - assert all( - py_coordinate not in list(release_matrix["lib3"].keys()) - for py_coordinate in py_coords_not_in_lib3 - ) - - -@pytest.mark.parametrize( - "matrix", - [({"py": ["3.8"], "rhel": ["7"]}), ({"py": ["3.8", "3.10"], "rhel": ["7", "8"]})], -) -def test_transpile_add_argument(tmpdir, matrix): - release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") - release_base = os.path.basename(release_file).strip(".yml") - with tmpdir.as_cwd(): - transpile_releases(release_file, os.getcwd(), matrix) - for rhel_coordinate in matrix["rhel"]: - rhel_coordinate_filename_format = f"rhel{rhel_coordinate}" - for py_coordinate in matrix["py"]: - py_coordinate_filename_format = f"py{py_coordinate.replace('.', '')}" - assert os.path.isfile( - f"{release_base}-{py_coordinate_filename_format}-{rhel_coordinate_filename_format}.yml" - ) - - -@pytest.mark.parametrize( - ("matrix", "error_message_content"), - [ - ({"py": ["3.8"], "rhel": ["7"]}, "Test passes, no error reported"), - ( - {"py": ["3.7"], "rhel": ["7"]}, - ["py37", "rhel7", "lib1"], - ), - ( - {"py": ["3.6"], "rhel": ["5"]}, - ["rhel5", "lib1"], - ), - ], - ids=["Pass for all packages", "Fail", "Fail"], -) -def test_check_version_exists_for_coordinates(matrix, error_message_content): - release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") - try: - transpile_releases(release_file, os.getcwd(), matrix) - except KeyError as exception_info: - assert all(word in str(exception_info) for word in error_message_content) - - -def test_get_py_coords(): - release_folder = os.path.join(_get_test_root(), "data", "test_releases") - release_base = "2020.01.a1" - py_coords = get_py_coords(release_base, release_folder) - assert py_coords == ["py27", "py36", "py38"] - - -def test_transpile_for_pip(tmpdir): - release_file = os.path.join(_get_test_root(), "data", "test_release_matrix.yml") - repo_file = os.path.join(_get_test_root(), "data", "test_repository.yml") - release_base = os.path.basename(release_file).strip(".yml") - not_pip_pkg = "lib3" - expected_line = "lib2==2.3.4" - versions_matrix = {"rhel": ["7"], "py": ["38"]} - with tmpdir.as_cwd(): - transpile_releases_for_pip( - release_file, - os.getcwd(), - repo_file, - versions_matrix, - ) - for rhel_ver in ("rhel7",): - for py_ver in ("py38",): - filename = f"{release_base}-{py_ver}-{rhel_ver}.req" - assert os.path.isfile(filename) - with open(filename, encoding="utf-8") as fil: - file_lines = fil.read().splitlines() - assert all(not line.startswith(not_pip_pkg) for line in file_lines) - assert expected_line in file_lines diff --git a/tests/test_release_transpiler_argument_types.py b/tests/test_release_transpiler_argument_types.py deleted file mode 100644 index 7e711758f..000000000 --- a/tests/test_release_transpiler_argument_types.py +++ /dev/null @@ -1,448 +0,0 @@ -import argparse -import sys -from contextlib import contextmanager -from os.path import abspath, dirname -from typing import List - -import pytest - -from komodo.release_transpiler import main as release_transpiler_main - -VALID_RELEASE_FOLDER = abspath(dirname(__file__) + "/data/test_releases") -VALID_RELEASE_BASE = "2020.01.a1" -VALID_OVERRIDE_MAPPING_FILE = abspath( - dirname(dirname(__file__)) + "/examples/stable.yml", -) -VALID_PYTHON_COORDS = "3.6,3.8" -VALID_MATRIX_FILE = abspath(dirname(__file__) + "/data/test_release_matrix.yml") -VALID_OUTPUT_FOLDER = abspath(dirname(__file__)) -VALID_MATRIX_COORDINATES = "{rhel: ['7'], py: ['3.8']}" - - -@contextmanager -def does_not_raise(): - yield - - -@pytest.mark.parametrize( - ("args", "expectation"), - [ - ( - [ - "--py_coords", - "3.8,3.6", - ], - does_not_raise(), - ), - ( - [], - does_not_raise(), - ), - ( - [ - "--py_coords", - "py3.8", - ], - pytest.raises(SystemExit), - ), - ( - [ - "--py_coords", - "test", - ], - pytest.raises(SystemExit), - ), - ( - [ - "--py_coords", - "'3.8, 3.6'", - ], - pytest.raises(SystemExit), - ), - ( - [ - "--py_coords", - "4.2", - ], - pytest.raises(SystemExit), - ), - ( - [ - "--py_coords", - "true", - ], - pytest.raises(SystemExit), - ), - ], -) -def test_combine_py_coords_type(args: List[str], expectation, monkeypatch): - monkeypatch.setattr( - sys, - "argv", - [ - "", - "combine", - "--release-folder", - VALID_RELEASE_FOLDER, - "--release-base", - VALID_RELEASE_BASE, - "--override-mapping", - VALID_OVERRIDE_MAPPING_FILE, - *args, - ], - ) - with expectation: - release_transpiler_main() - - -@pytest.mark.parametrize( - ("args", "expectation"), - [ - ( - [ - "--release-folder", - VALID_RELEASE_FOLDER, - "--py_coords", - "3.8,3.6", - ], - does_not_raise(), - ), - ( - [ - "--release-folder", - VALID_RELEASE_FOLDER, - ], - does_not_raise(), - ), - ( - [ - "--release-folder", - "FOLDER/DOES/NOT/EXIST", - ], - pytest.raises(NotADirectoryError), - ), - ( - [ - "--release-folder", - "true", - ], - pytest.raises(NotADirectoryError), - ), - ( - [ - "--release-folder", - "null", - ], - pytest.raises(NotADirectoryError), - ), - ( - [ - "--release-folder", - "random", - "--override-mapping", - VALID_OVERRIDE_MAPPING_FILE, - ], - pytest.raises(NotADirectoryError), - ), - ], -) -def test_combine_py_release_folder_type(args: List[str], expectation, monkeypatch): - monkeypatch.setattr( - sys, - "argv", - [ - "", - "combine", - "--release-base", - VALID_RELEASE_BASE, - "--override-mapping", - VALID_OVERRIDE_MAPPING_FILE, - *args, - ], - ) - with expectation: - release_transpiler_main() - - -@pytest.mark.parametrize( - ("args", "expectation"), - [ - ( - [ - "--override-mapping", - VALID_OVERRIDE_MAPPING_FILE, - "--py_coords", - "3.8,3.6", - ], - does_not_raise(), - ), - ( - [ - "--override-mapping", - VALID_OVERRIDE_MAPPING_FILE, - ], - does_not_raise(), - ), - ( - [ - "--override-mapping", - "FAKE/PATH/mapping.yml", - ], - pytest.raises(FileNotFoundError), - ), - ( - [ - "--override-mapping", - "fake_mapping.yml", - ], - pytest.raises(FileNotFoundError), - ), - ( - [ - "--override-mapping", - "random", - ], - pytest.raises(FileNotFoundError), - ), - ( - [ - "--override-mapping", - "0", - ], - pytest.raises(FileNotFoundError), - ), - ], -) -def test_combine_py_override_mapping_type(args: List[str], expectation, monkeypatch): - monkeypatch.setattr( - sys, - "argv", - [ - "", - "combine", - "--release-base", - VALID_RELEASE_BASE, - "--release-folder", - VALID_RELEASE_FOLDER, - *args, - ], - ) - - with expectation: - release_transpiler_main() - - -@pytest.mark.parametrize( - ("args", "expectation"), - [ - ( - [ - "--release-base", - VALID_RELEASE_BASE, - ], - does_not_raise(), - ), - ( - [ - "--release-base", - "INVALID_RELEASE_BASE", - ], - pytest.raises(argparse.ArgumentTypeError), - ), - ], -) -def test_combine_py_release_base_type(args: List[str], expectation, monkeypatch): - monkeypatch.setattr( - sys, - "argv", - [ - "", - "combine", - "--release-folder", - VALID_RELEASE_FOLDER, - "--override-mapping", - VALID_OVERRIDE_MAPPING_FILE, - "--py_coords", - "3.8,3.6", - *args, - ], - ) - with expectation: - release_transpiler_main() - - -@pytest.mark.parametrize( - ("args", "expectation"), - [ - ( - [ - "--matrix-file", - VALID_MATRIX_FILE, - "--matrix-coordinates", - VALID_MATRIX_COORDINATES, - ], - does_not_raise(), - ), - ( - [ - "--matrix-file", - VALID_MATRIX_FILE, - ], - does_not_raise(), - ), - ( - [ - "--matrix-file", - dirname(VALID_MATRIX_FILE), - ], - pytest.raises(FileNotFoundError), - ), - ( - [ - "--matrix-file", - f"{VALID_MATRIX_FILE}/does_not_exist.yml", - ], - pytest.raises(FileNotFoundError), - ), - ( - [ - "--matrix-file", - "random_string", - ], - pytest.raises(FileNotFoundError), - ), - ( - [ - "--matrix-file", - "null", - ], - pytest.raises(FileNotFoundError), - ), - ], -) -def test_transpile_py_matrix_file_type(args: List[str], expectation, monkeypatch): - monkeypatch.setattr( - sys, - "argv", - ["", "transpile", "--output-folder", VALID_RELEASE_FOLDER, *args], - ) - with expectation: - release_transpiler_main() - - -@pytest.mark.parametrize( - ("args", "expectation"), - [ - ( - [ - "--output-folder", - VALID_RELEASE_FOLDER, - "--matrix-coordinates", - VALID_MATRIX_COORDINATES, - ], - does_not_raise(), - ), - ( - [ - "--output-folder", - VALID_RELEASE_FOLDER, - ], - does_not_raise(), - ), - ( - [ - "--output-folder", - f"{VALID_RELEASE_FOLDER}/does_not_exist", - ], - pytest.raises(NotADirectoryError), - ), - ( - [ - "--output-folder", - "random_string", - ], - pytest.raises(NotADirectoryError), - ), - ( - [ - "--output-folder", - "null", - ], - pytest.raises(NotADirectoryError), - ), - ( - [ - "--output-folder", - "0", - ], - pytest.raises(NotADirectoryError), - ), - ], -) -def test_transpile_py_output_folder_type(args: List[str], expectation, monkeypatch): - monkeypatch.setattr( - sys, - "argv", - ["", "transpile", "--matrix-file", VALID_MATRIX_FILE, *args], - ) - with expectation: - release_transpiler_main() - - -@pytest.mark.parametrize( - ("args", "expectation"), - [ - ( - [ - "--matrix-coordinates", - VALID_MATRIX_COORDINATES, - ], - does_not_raise(), - ), - ( - [], - does_not_raise(), - ), - ( - [ - "--matrix-coordinates", - "0", - ], - pytest.raises(TypeError), - ), - ( - [ - "--matrix-coordinates", - "random_string", - ], - pytest.raises(TypeError), - ), - ( - [ - "--matrix-coordinates", - "false", - ], - pytest.raises(TypeError), - ), - ], -) -def test_transpile_py_matrix_coordinates_type( - args: List[str], - expectation, - monkeypatch, -): - monkeypatch.setattr( - sys, - "argv", - [ - "", - "transpile", - "--matrix-file", - VALID_MATRIX_FILE, - "--output-folder", - VALID_RELEASE_FOLDER, - *args, - ], - ) - with expectation: - release_transpiler_main() diff --git a/tests/test_switch.py b/tests/test_switch.py deleted file mode 100644 index 71d783938..000000000 --- a/tests/test_switch.py +++ /dev/null @@ -1,58 +0,0 @@ -import pytest - -from komodo import switch -from komodo.data import Data -from komodo.switch import MIGRATION_WARNING - - -def test_write_activator_switches(tmpdir): - prefix = tmpdir / "prefix" - release = "2020.01.01-py27-rhel6" - expected_release = "2020.01.01-py27" - switch.create_activator_switch(Data(), prefix, release) - - actual_bash_activator = prefix / f"{expected_release}/enable" - assert ( - actual_bash_activator.read_text(encoding="utf-8").strip() - == f""" -if [[ $(uname -r) == *el7* ]] ; then - export KOMODO_ROOT={prefix} - KOMODO_RELEASE_REAL={expected_release} - - source $KOMODO_ROOT/$KOMODO_RELEASE_REAL-rhel7/enable - export PS1="(${{KOMODO_RELEASE_REAL}}) ${{_PRE_KOMODO_PS1}}" - export KOMODO_RELEASE=$KOMODO_RELEASE_REAL -else - echo -e "{MIGRATION_WARNING}" -fi -""".strip() - ) - - actual_csh_activator = prefix / f"{expected_release}/enable.csh" - assert ( - actual_csh_activator.read_text(encoding="utf-8").strip() - == f""" -if ( `uname -r` =~ *el7* ) then - setenv KOMODO_ROOT {prefix} - set KOMODO_RELEASE_REAL = "{expected_release}" - - source $KOMODO_ROOT/$KOMODO_RELEASE_REAL-rhel7/enable.csh - if ( $?_KOMODO_OLD_PROMPT ) then - set prompt = "[$KOMODO_RELEASE_REAL] $_KOMODO_OLD_PROMPT" - endif - setenv KOMODO_RELEASE $KOMODO_RELEASE_REAL -else - echo -e "{MIGRATION_WARNING}" -endif -""".strip() - ) - - -def test_write_activator_switches_for_non_matrix_build(tmpdir): - prefix = tmpdir / "prefix" - release = "foobar" - - try: - switch.create_activator_switch(Data(), prefix, release) - except ValueError as e: - pytest.fail("Unexpected ValueError " + e) diff --git a/tests/test_yaml_file_types.py b/tests/test_yaml_file_types.py index ba6cee9f7..26c9a3bde 100644 --- a/tests/test_yaml_file_types.py +++ b/tests/test_yaml_file_types.py @@ -71,12 +71,9 @@ def test_release_file_yaml_type(content, expectations): "valid", ( "bleeding-py36.yml", - "/home/anyuser/komodo/2020.01.03-py36-rhel6.yml", + "/home/anyuser/komodo/2020.01.03-py36.yml", "myrelease-py36.yml", "myrelease-py310.yml", - "myrelease-py310-rhel8.yml", - "myrelease-py36-rhel6.yml", - "myrelease-py36-rhel7.yml", ), ) def test_release_name_valid(valid):