From deb8059a422d52622c492702b1a17fdf7cbc96c0 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 1 Nov 2024 10:15:59 -0500 Subject: [PATCH 1/3] rearrange(worker): small refactoring of key_source_sql construction --- .../dj_worker/worker_schema.py | 47 ++++++++++++++----- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/datajoint_utilities/dj_worker/worker_schema.py b/datajoint_utilities/dj_worker/worker_schema.py index cda999d..b65e093 100644 --- a/datajoint_utilities/dj_worker/worker_schema.py +++ b/datajoint_utilities/dj_worker/worker_schema.py @@ -38,19 +38,33 @@ class Process(dj.Part): """ @classmethod - def get_workers_progress(cls): + def get_workers_progress(cls, worker_name=None, process_name=None): """ Return the operation progress for all registered workers (jobs status for each AutoPopulate process) - :return: pandas DataFrame of workers jobs status + + Args: + worker_name (str): name of the worker (optionally restrict by worker_name) + process_name (str): name of the process (optionally restrict by process_name) + + Returns: + pandas DataFrame of workers jobs status """ + restriction = {} + if worker_name: + restriction["worker_name"] = worker_name + if process_name: + restriction["process"] = process_name + workflow_status = ( - (cls.Process & "key_source_sql is not NULL") - .proj( - "process", - "key_source_sql", - table_name="full_table_name", - total="NULL", - incomplete="NULL", + ( + (cls.Process & "key_source_sql is not NULL").proj( + "process", + "key_source_sql", + table_name="full_table_name", + total="NULL", + incomplete="NULL", + ) + & restriction ) .fetch(format="frame") .reset_index() @@ -96,7 +110,7 @@ def get_workers_progress(cls): ( workflow_status.loc[r_idx, "total"], workflow_status.loc[r_idx, "incomplete"], - ) = cls._get_key_source_count(r.key_source_sql, r.table_name) + ) = cls.get_key_source_count(r.key_source_sql, r.table_name) # merge key_source and jobs status workflow_status.set_index("table_name", inplace=True) @@ -124,7 +138,10 @@ def get_workers_progress(cls): return workflow_status @classmethod - def _get_key_source_count(cls, key_source_sql, target_full_table_name): + def get_incomplete_key_source_sql(cls, key_source_sql, target_full_table_name): + """ + From `key_source_sql`, build a SQL statement to find incomplete key_source entries in the target table + """ def _rename_attributes(table, props): return ( table.proj( @@ -169,6 +186,14 @@ def _remove_enclosed_parentheses(input_string): key_source_sql + f"{AND_or_WHERE}(({ks_attrs_sql}) not in (SELECT {ks_attrs_sql} FROM {target.full_table_name}))" ) + return incomplete_sql + + @classmethod + def get_key_source_count(cls, key_source_sql, target_full_table_name): + """ + From `key_source_sql`, count the total and incomplete key_source entries in the target table + """ + incomplete_sql = cls.get_incomplete_key_source_sql(key_source_sql, target_full_table_name) try: total = len(dj.conn().query(key_source_sql).fetchall()) incomplete = len(dj.conn().query(incomplete_sql).fetchall()) From a1556da3e56c2053db880925819f52b114a4300b Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 1 Nov 2024 10:16:24 -0500 Subject: [PATCH 2/3] update: version --- datajoint_utilities/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint_utilities/version.py b/datajoint_utilities/version.py index 318428b..9a54cb4 100644 --- a/datajoint_utilities/version.py +++ b/datajoint_utilities/version.py @@ -1,3 +1,3 @@ """Package metadata.""" -__version__ = "0.4.0" +__version__ = "0.4.1" From 75a486ca46422f668024149bce46014454a0a7a6 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Fri, 8 Nov 2024 12:07:22 -0600 Subject: [PATCH 3/3] Update: use pyproject.toml, version 0.5.0 --- CHANGELOG.md | 13 ++++++++ datajoint_utilities/version.py | 2 +- pyproject.toml | 61 ++++++++++++++++++++++++++++++++++ requirements.txt | 6 ---- setup.py | 36 -------------------- 5 files changed, 75 insertions(+), 43 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9404018 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changelog + +Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. + +## [0.5.0] - 2024-11-08 + +### Added + +- Started CHANGELOG +- Install with `pyproject.toml` + + +[0.0.0]: https://github.com/datajoint-company/datajoint-utilities/releases/tag/0.5.0 diff --git a/datajoint_utilities/version.py b/datajoint_utilities/version.py index 9a54cb4..3bbff80 100644 --- a/datajoint_utilities/version.py +++ b/datajoint_utilities/version.py @@ -1,3 +1,3 @@ """Package metadata.""" -__version__ = "0.4.1" +__version__ = "0.5.0" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..be35fcd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools>=62.0", "wheel>=0.37"] +build-backend = "setuptools.build_meta" + +[project] +name = "datajoint-utilities" +version = "0.5.0" +description = "A general purpose repository containing all generic tools/utilities surrounding the DataJoint ecosystem" +requires-python = ">=3.9, <3.12" +license = { file = "LICENSE" } +authors = [{ name = "DataJoint", email = "info@datajoint.com" }] +keywords = ["datajoint", "workflow"] +classifiers = [ + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries :: Python Modules", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "datajoint>=0.14.3", + "termcolor", + "slack-sdk", + "python-dotenv", + "boto3", + "requests", +] + +[project.scripts] +tmplcfg = "datajoint_utilities.cmdline.tmplcfg:cli" + +[project.urls] +Source = "https://github.com/datajoint-company/datajoint-utilities" +DataJoint = "https://datajoint.com/docs" + +[tool.setuptools] +package-data = { "*" = ["*.pyi", "py.typed"] } + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-rA" +testpaths = ["tests"] + +[tool.black] +line-length = 88 +target-version = ["py310"] +color = false +exclude = ''' +/( + \.git + | \.venv + | _build + | example_data + | build + | dist + | env + | venv +)/ +''' + +[tool.isort] +profile = "black" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a893e0c..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -datajoint>=0.14.3 -termcolor -slack-sdk -python-dotenv -boto3 -requests diff --git a/setup.py b/setup.py deleted file mode 100644 index b74882d..0000000 --- a/setup.py +++ /dev/null @@ -1,36 +0,0 @@ -import pathlib - -import setuptools - -pkg_name = "datajoint_utilities" - -with open("README.md", "r") as fh: - long_description = fh.read() - -with open(pathlib.Path(__file__).parent / pkg_name / "version.py") as f: - exec(f.read()) - -with open(pathlib.Path(__file__).parent / "requirements.txt") as f: - requirements = f.read().splitlines() - -setuptools.setup( - name=pkg_name.replace("_", "-"), - version=__version__, - author="Thinh Nguyen", - author_email="thinh@datajoint.com", - description="Unofficial utilities to support the DataJoint framework.", - long_description=long_description, - long_description_content_type="text/markdown", - url=f"https://github.com/datajoint-company/{pkg_name.replace('_', '-')}", - packages=setuptools.find_packages(exclude=["contrib", "docs", "tests*"]), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - entry_points={ - "console_scripts": ("tmplcfg=datajoint_utilities.cmdline.tmplcfg:cli",), - }, - install_requires=requirements, - zip_safe=False, -)