0.3.10 (#138)

* ⬆️ Upgrade datar to 0.11 and format test files

* ✨ Add cli command version to show versions of deps

* ➖ Remove rich as it is required by xqute already

* 📝 Update docs for cli command `version`

* 🔖 0.3.10

docs/CHANGELOG.md

@@ -1,9 +1,16 @@
 # Change Log
 
+## 0.3.10
+
+- ⬆️ Upgrade xqute to 0.1.3
+- ⬆️ Upgrade datar to 0.11 and format test files
+- ✨ Add cli command version to show versions of deps
+- ➖ Remove rich as it is required by xqute already
+
 ## 0.3.9
 
-- ⬆️ Pump pipda to 0.11
-- ⬆️ Pump xqute to 0.1.2
+- ⬆️ Bump pipda to 0.11
+- ⬆️ Bump xqute to 0.1.2
 
 ## 0.3.8
 

docs/cli.md

@@ -2,7 +2,7 @@
 
 To run it:
 
-```
+```shell
 ❯ pipen
 
 DESCRIPTION:
@@ -15,6 +15,7 @@ OPTIONAL OPTIONS:
   -h, --help                      - Print help information for the CLI tool.
 
 COMMANDS:
+  version                         - Print versions of pipen and its dependencies
   profile                         - List available profiles.
   plugins                         - List installed plugins
   help                            - Print help for commands
@@ -57,6 +58,9 @@ It is used to list the configurations/profiles in current directory. Run `pipen
 
 This subcommand is used to list the plugins for `pipen` itself, templates, scheduler and cli. Run `pipen plugins` or `pipen help plugins` to get more information.
 
+## The `version` subcommand
+
+This command prints the versions of `pipen` and its dependencies.
 
 [1]: https://github.com/pwwang/pyparam
 [2]: ../plugin

pipen/cli/version.py

@@ -0,0 +1,61 @@
+"""Print help for commands"""
+from typing import Any, Mapping
+
+from rich import print
+from pyparam import Params
+
+from ._hooks import CLIPlugin
+
+__all__ = ("CLIVersionPlugin",)
+
+
+def get_pkg_version(pkg: str) -> str:
+    try:
+        from importlib.metadata import version
+        return version(pkg)
+    except ImportError:  # pragma: no cover
+        from pkg_resources import get_distribution  # type: ignore
+        return get_distribution(pkg).version
+
+
+class CLIVersionPlugin(CLIPlugin):
+    """Print versions of pipen and its dependencies"""
+
+    name = "version"
+
+    @property
+    def params(self) -> Params:
+        """Define the params"""
+        pms = Params(
+            desc=self.__class__.__doc__,
+            help_on_void=False,
+        )
+        return pms
+
+    def exec_command(self, args: Mapping[str, Any]) -> None:
+        """Run the command"""
+        import sys
+        from .. import __version__
+
+        versions = {"python": sys.version, "pipen": __version__}
+
+        for pkg in (
+            "liquidpy",
+            "pandas",
+            "python-slugify",
+            "enlighten",
+            "pyparam",
+            "xqute",
+            "python-simpleconf",
+            "pipda",
+            "varname",
+        ):
+            versions[pkg] = get_pkg_version(pkg)
+
+        keylen = max(map(len, versions))
+        for key in versions:
+            ver = versions[key]
+            verlines = ver.splitlines()
+            print(f"{key.ljust(keylen)}: {verlines.pop(0)}")
+            for verline in verlines:
+                print(f"{' ' * keylen}  {verline}")

pipen/proc.py

@@ -503,7 +503,7 @@ def _compute_requires(
         # req.nexts
         my_nexts = None if cls.nexts is None else cls.nexts[:]
         for req in requires:  # type: ignore
-            if req.nexts is None:
+            if not req.nexts:
                 req.nexts = [cls]
             else:
                 req.nexts.append(cls)  # type: ignore

pipen/version.py

@@ -1,3 +1,3 @@
 """Provide version of pipen"""
 
-__version__ = "0.3.9"
+__version__ = "0.3.10"

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.masonry.api"
 
 [tool.poetry]
 name = "pipen"
-version = "0.3.9"
+version = "0.3.10"
 description = "A pipeline framework for python"
 authors = [ "pwwang <[email protected]>",]
 license = "MIT"
@@ -16,14 +16,14 @@ repository = "https://github.com/pwwang/pipen"
 python = "^3.7.1" # align with datar/pandas
 liquidpy = "^0.7"
 pandas = "^1.3"
-rich = "^12"
 python-slugify = "^6"
 enlighten = "^1"
 cached-property = "^1"
 more-itertools = "^8"
 pyparam = "^0.5"
 xqute = "^0.1"
 ## included in xqute
+# rich = "^12"
 # diot = "^0.1"
 # simplug = "^0.0"
 ## including rtoml
@@ -35,17 +35,20 @@ varname = "^0.10"
 openpyxl = "^3"
 pytest = "^7"
 pytest-cov = "^4"
-pytest-xdist = "^2"
-datar = { version = "^0.10", extras = ["pandas"] }
+pytest-xdist = "^3"
+datar = { version = "^0.11", extras = ["pandas"] }
 
 [tool.poetry.scripts]
 pipen = "pipen.cli:main"
 
 [tool.pytest.ini_options]
-addopts = "-vv -n auto -p no:asyncio -W error::UserWarning --cov-config=.coveragerc --cov=pipen --cov-report xml:.coverage.xml --cov-report term-missing"
+addopts = "-vv -n auto -p no:asyncio -p no:benchmark -W error::UserWarning --cov-config=.coveragerc --cov=pipen --cov-report xml:.coverage.xml --cov-report term-missing"
 # addopts = "-vv -n auto -p no:asyncio -W error::UserWarning"
 console_output_style = "progress"
 junit_family = "xunit1"
+filterwarnings = [
+   "ignore:The --rsyncdir command line argument and rsyncdirs config variable are deprecated.:DeprecationWarning",
+]
 
 [tool.mypy]
 ignore_missing_imports = true

tests/conftest.py

@@ -0,0 +1,3 @@
+import datar
+
+datar.options(backends=["numpy", "pandas"])

tests/helpers.py

@@ -1,9 +1,9 @@
 import sys
-import signal
+# import signal
 from tempfile import gettempdir
 from pathlib import Path
-from shutil import rmtree
-from multiprocessing import Process
+# from shutil import rmtree
+# from multiprocessing import Process
 
 import pytest
 from pipen import Proc, Pipen, plugin
@@ -76,7 +76,10 @@ class RetryProc(ErrorProc):
     error_strategy = "retry"
     num_retries = 10
     lang = sys.executable  # python
-    script = "import sys, time; sys.exit(1 if time.time() < {{in.starttime}} + 3 else 0)"
+    script = """
+        import sys, time
+        sys.exit(1 if time.time() < {{in.starttime}} + 3 else 0)
+    """
 
 
 class OutputRenderErrorProc(Proc):
@@ -124,6 +127,7 @@ class FileInputProc(Proc):
     output = "out:file:{{in.in.split('/')[-1]}}"
     script = "cat {{in.in}} > {{out.out}}"
 
+
 class OutputNotGeneratedProc(Proc):
     """Process with output file not generated intentionally"""
 
@@ -157,11 +161,11 @@ class DirOutputProc(Proc):
 
 
 class SimplePlugin:
-
     @plugin.impl
     async def on_init(pipen):
         print("SimplePlugin")
 
+
 @pytest.fixture
 def pipen(tmp_path):
     """Get a simple Pipen object each time"""
@@ -177,6 +181,7 @@ def pipen(tmp_path):
 
     return pipen_simple
 
+
 @pytest.fixture
 def pipen_with_plugin(tmp_path):
     """Get a simple Pipen object each time"""

tests/test_channel.py

@@ -1,53 +1,58 @@
 from io import StringIO
 from pathlib import Path
 from math import ceil
-import pytest
-from pipen.channel import *
-from datar.all import *
+
+import pytest  # noqa
+from pipen.channel import Channel, expand_dir, collapse_files
+from datar.tibble import tibble
 
 from pandas import DataFrame
 
+
 def test_create():
     assert isinstance(Channel.create(DataFrame([[1]])), DataFrame)
 
+
 def test_from_glob():
-    glob = Path(__file__).parent / 'test_*.py'
-    glob_files = list(Path(__file__).parent.glob('test_*.py'))
+    glob = Path(__file__).parent / "test_*.py"
+    glob_files = list(Path(__file__).parent.glob("test_*.py"))
     ch = Channel.from_glob(glob)
     assert ch.shape == (len(glob_files), 1)
 
+
 def test_from_pairs():
-    glob = Path(__file__).parent / 'test_*.py'
-    glob_files = list(Path(__file__).parent.glob('test_*.py'))
+    glob = Path(__file__).parent / "test_*.py"
+    glob_files = list(Path(__file__).parent.glob("test_*.py"))
     ch = Channel.from_pairs(glob)
     assert ch.shape == (ceil(len(glob_files) / 2.0), 2)
 
+
 def test_expand_dir_collapse_files():
     ch0 = Channel.create([(Path(__file__).parent.as_posix(), 1)])
-    ch1 = ch0 >> expand_dir(pattern='test_*.py')
-    glob_files = list(Path(__file__).parent.glob('test_*.py'))
+    ch1 = ch0 >> expand_dir(pattern="test_*.py")
+    glob_files = list(Path(__file__).parent.glob("test_*.py"))
     assert ch1.shape == (len(glob_files), 2)
 
     ch2 = ch1 >> collapse_files()
     assert ch2.equals(ch0)
 
+
 def test_from_csv(tmp_path):
-    df = tibble(a=[1,2], b=[3,4])
-    df.to_csv(tmp_path / 'input.csv', index=False)
-    ch = Channel.from_csv(tmp_path / 'input.csv')
+    df = tibble(a=[1, 2], b=[3, 4])
+    df.to_csv(tmp_path / "input.csv", index=False)
+    ch = Channel.from_csv(tmp_path / "input.csv")
     assert ch.equals(df)
 
+
 def test_from_excel(tmp_path):
-    df = tibble(a=[1,2], b=[3,4])
-    df.to_excel(tmp_path / 'input.xls', index=False)
-    ch = Channel.from_excel(tmp_path / 'input.xls')
+    df = tibble(a=[1, 2], b=[3, 4])
+    df.to_excel(tmp_path / "input.xls", index=False)
+    ch = Channel.from_excel(tmp_path / "input.xls")
     assert ch.equals(df)
 
+
 def test_from_table():
-    df = StringIO("""a b
-1 3
-2 4
-""")
+    df = StringIO("a b\n1 3\n2 4\n")
     ch = Channel.from_table(df, sep=" ")
-    exp = tibble(a=[1,2], b=[3,4])
-    assert ch.equals(exp)
+    exp = tibble(a=[1, 2], b=[3, 4])
+    assert ch.equals(exp)

tests/test_cli.py

@@ -1,5 +1,6 @@
 from subprocess import check_output, CalledProcessError
-import pytest
+import pytest  # noqa: F401
+
 
 def cmdoutput(cmd):
     try:
@@ -15,16 +16,19 @@ def test_main():
     out = cmdoutput(["pipen", "--help"])
     assert "CLI Tool for pipen" in out
 
+
 def test_nosuch_command():
     out = cmdoutput(["pipen", "x"])
     assert "No such command" in out
 
+
 def test_help():
     out = cmdoutput(["pipen", "help", "x"])
     assert "No such command" in out
     out = cmdoutput(["pipen", "help", "profile"])
     assert "List available profiles" in out
 
+
 def test_profile_all():
     out = cmdoutput(["pipen", "profile"])
     assert "Note:" in out
@@ -38,3 +42,10 @@ def test_profile_default():
 def test_profile_nosuch():
     out = cmdoutput(["pipen", "profile", "-n", "nosuch"])
     assert "Profile: nosuch" not in out
+
+
+def test_version():
+    out = cmdoutput(["pipen", "version"])
+    assert "pipen" in out
+    assert "python" in out
+    assert "liquidpy" in out

tests/test_job.py

@@ -117,7 +117,7 @@ def test_check_cached_input_or_output_different(
 
     caplog.clear()
     pipen.set_starts(proc_io_diff2).run()
-    pipen.build_proc_relationships() # not redoing it.
+    pipen.build_proc_relationships()  # not redoing it.
     assert "Not cached (input or output is different)" in caplog.text
 
 
@@ -175,7 +175,9 @@ def test_check_cached_force_cache(caplog, pipen, infile):
 
 
 def test_check_cached_infile_newer(caplog, pipen, infile):
-    proc_infile_newer = Proc.from_proc(MixedInputProc, input_data=[(1, infile)])
+    proc_infile_newer = Proc.from_proc(
+        MixedInputProc, input_data=[(1, infile)]
+    )
     pipen.set_starts(proc_infile_newer).run()
 
     caplog.clear()

tests/test_pipen.py

@@ -58,8 +58,10 @@ def test_cyclic_dependency(pipen):
 def test_not_cyclic_for_subclass_of_proc_in_pipeline(pipen):
     proc1 = Proc.from_proc(NormalProc, input_data=[1])
     proc2 = Proc.from_proc(NormalProc, requires=proc1)
+
     class proc3(proc1):
         requires = proc2
+
     pipen.set_starts(proc1).run()
     assert pipen.procs == [proc1, proc2, proc3]
 

tests/test_plugin.py

@@ -7,6 +7,7 @@
 
 from .helpers import pipen, OutputNotGeneratedProc
 
+
 def test_job_succeeded(pipen, caplog):
 
     out = pipen.set_starts(OutputNotGeneratedProc).run()