From b740a2f781c6d2df391c93ed5e792620a458ced5 Mon Sep 17 00:00:00 2001 From: Sebastian Tramp Date: Wed, 3 Apr 2024 17:54:21 +0200 Subject: [PATCH] operate all plugin specifica from template out --- .idea/.gitignore | 8 ++ .idea/imr.iml | 17 ++++ Taskfile.yaml | 11 +-- cmem_plugin_imr/__init__.py | 1 - cmem_plugin_imr/example_icon.svg | 83 ----------------- cmem_plugin_imr/example_transform.py | 74 ---------------- cmem_plugin_imr/example_workflow.py | 90 ------------------- imr/__init__.py | 6 ++ poetry.lock | 127 +-------------------------- pyproject.toml | 10 +-- tests/test_example.py | 79 ++--------------- 11 files changed, 40 insertions(+), 466 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/imr.iml delete mode 100644 cmem_plugin_imr/__init__.py delete mode 100644 cmem_plugin_imr/example_icon.svg delete mode 100644 cmem_plugin_imr/example_transform.py delete mode 100644 cmem_plugin_imr/example_workflow.py create mode 100644 imr/__init__.py diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/imr.iml b/.idea/imr.iml new file mode 100644 index 0000000..e85cfa8 --- /dev/null +++ b/.idea/imr.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Taskfile.yaml b/Taskfile.yaml index 3d8ab52..56c1eda 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -10,7 +10,7 @@ dotenv: ['.copier-answers.env', '.env'] - check:prepare vars: - PACKAGE: cmem_plugin_$project_slug + PACKAGE: $project_slug DIST_DIR: dist includes: @@ -160,15 +160,6 @@ tasks: # }}} # {{{ build and deploy tasks - - deploy: - desc: Install plugin package in Corporate Memory - deps: - - build - cmds: - - cmemc admin workspace python install dist/*.tar.gz - - cmemc admin workspace python list-plugins - build: desc: Build a tarball and a wheel package <<: *preparation diff --git a/cmem_plugin_imr/__init__.py b/cmem_plugin_imr/__init__.py deleted file mode 100644 index e215ab0..0000000 --- a/cmem_plugin_imr/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""imr - main package""" diff --git a/cmem_plugin_imr/example_icon.svg b/cmem_plugin_imr/example_icon.svg deleted file mode 100644 index 4710e0b..0000000 --- a/cmem_plugin_imr/example_icon.svg +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/cmem_plugin_imr/example_transform.py b/cmem_plugin_imr/example_transform.py deleted file mode 100644 index e561948..0000000 --- a/cmem_plugin_imr/example_transform.py +++ /dev/null @@ -1,74 +0,0 @@ -"""lifetime(age) transform plugin module""" -import datetime -from collections.abc import Sequence -from datetime import date - -from cmem_plugin_base.dataintegration.description import ( - Plugin, - PluginParameter, -) -from cmem_plugin_base.dataintegration.plugins import TransformPlugin - - -@Plugin( - label="Lifetime (age - example from template)", - description="From the input date," - "the value gets transformed into number of years (age)." - " Supports only xsd:date(YYYY-MM-DD) format.", - documentation=""" -This example transform operator returns lifetime(age). - -From the input date, -the value gets transformed into number of years(age). - -Input Date: 2000-05-22 -Current Date: 2022-08-19 -Transformed Output: 22 - -The parameter can be specified: - -- 'date': specify a date in xsd:date(yyyy-MM-dd) format -""", - parameters=[ - PluginParameter( - name="start_date", - label="Date", - description="specify a date to know its lifetime(age).", - default_value=None, - ), - ], -) -class Lifetime(TransformPlugin): - """Lifetime Transform Plugin""" - - DATE_FORMAT = "%Y-%m-%d" - - def __init__(self, start_date: str): - self.start_date = start_date - - def transform(self, inputs: Sequence[Sequence[str]]) -> Sequence[str]: - """Do the actual transformation of values""" - result = [] - if len(inputs) != 0: - for collection in inputs: - result += [f"{self._calculate_age(_)}" for _ in collection] - if len(result) == 0 and len(self.start_date) > 0: - result += [f"{self._calculate_age(self.start_date)}"] - return result - - def _calculate_age(self, value: str) -> int: - """Calculate age in years""" - today = date.today() # noqa: DTZ011 - born = datetime.datetime.strptime(value, self.DATE_FORMAT).date() # noqa: DTZ007 - try: - birthday = born.replace(year=today.year) - - # raised when birthdate is February 29 - # and the current year is not a leap year - except ValueError: - birthday = born.replace(year=today.year, month=born.month + 1, day=1) - - if birthday > today: - return today.year - born.year - 1 - - return today.year - born.year diff --git a/cmem_plugin_imr/example_workflow.py b/cmem_plugin_imr/example_workflow.py deleted file mode 100644 index 31d4058..0000000 --- a/cmem_plugin_imr/example_workflow.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Random values workflow plugin module""" -import uuid -from collections.abc import Sequence -from secrets import token_urlsafe - -from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport -from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter -from cmem_plugin_base.dataintegration.entity import ( - Entities, - Entity, - EntityPath, - EntitySchema, -) -from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin - - -@Plugin( - label="Random Values (example from template)", - description="Generates random values of X rows a Y values.", - documentation=""" -This example workflow operator generates random values. - -The values are generated in X rows a Y values. Both parameter can be specified: - -- 'number_of_entities': How many rows do you need. -- 'number_of_values': How many values per row do you need. -""", - icon=Icon(package=__package__, file_name="example_icon.svg"), - parameters=[ - PluginParameter( - name="number_of_entities", - label="Entities (Rows)", - description="How many rows will be created per run.", - default_value="10", - ), - PluginParameter( - name="number_of_values", - label="Values (Columns)", - description="How many values are created per entity / row.", - default_value="5", - ), - ], -) -class DollyPlugin(WorkflowPlugin): - """Example Workflow Plugin: Random Values""" - - def __init__(self, number_of_entities: int = 10, number_of_values: int = 5) -> None: - if number_of_entities < 1: - raise ValueError("Entities (Rows) needs to be a positive integer.") - - if number_of_values < 1: - raise ValueError("Values (Columns) needs to be a positive integer.") - - self.number_of_entities = number_of_entities - self.number_of_values = number_of_values - - def execute( - self, - inputs: Sequence[Entities], # noqa: ARG002 - context: ExecutionContext, - ) -> Entities: - """Run the workflow operator.""" - self.log.info("Start creating random values.") - self.log.info(f"Config length: {len(self.config.get())}") - value_counter = 0 - entities = [] - for _ in range(self.number_of_entities): - entity_uri = f"urn:uuid:{uuid.uuid4()!s}" - values = [] - for _ in range(self.number_of_values): - values.append([token_urlsafe(16)]) - value_counter += 1 - context.report.update( - ExecutionReport( - entity_count=value_counter, - operation="wait", - operation_desc="random values generated", - ) - ) - entities.append(Entity(uri=entity_uri, values=values)) - paths = [] - for path_no in range(self.number_of_values): - path_uri = f"https://example.org/vocab/RandomValuePath/{path_no}" - paths.append(EntityPath(path=path_uri)) - schema = EntitySchema( - type_uri="https://example.org/vocab/RandomValueRow", - paths=paths, - ) - self.log.info(f"Happy to serve {value_counter} random values.") - return Entities(entities=entities, schema=schema) diff --git a/imr/__init__.py b/imr/__init__.py new file mode 100644 index 0000000..1f22bfe --- /dev/null +++ b/imr/__init__.py @@ -0,0 +1,6 @@ +"""imr - main package""" + + +def dummy_function(var: int) -> int: + """Add one to var :-)""" + return var + 1 diff --git a/poetry.lock b/poetry.lock index 3418840..d8f2686 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "certifi" @@ -124,39 +124,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "cmem-cmempy" -version = "24.1.0" -description = "API for eccenca Corporate Memory" -optional = false -python-versions = ">=3.9,<4.0" -files = [ - {file = "cmem_cmempy-24.1.0-py3-none-any.whl", hash = "sha256:1bda062e293b678afac574dd92b517141a52d43a4ec6c90eab9f15421aca5bda"}, - {file = "cmem_cmempy-24.1.0.tar.gz", hash = "sha256:a85f427d27ed9fee9e4e2aae1737cd06f60df2da8a5c89bd0d8634d877a26e86"}, -] - -[package.dependencies] -certifi = ">=2023.7.22" -pyparsing = ">=3.1.1,<4.0.0" -rdflib = ">=6.3.2,<7.0.0" -requests = ">=2.31.0,<3.0.0" -requests-toolbelt = ">=1.0.0,<2.0.0" - -[[package]] -name = "cmem-plugin-base" -version = "4.5.0" -description = "Base classes for developing eccenca Coporate Memory plugins." -optional = false -python-versions = ">=3.11,<4.0" -files = [ - {file = "cmem_plugin_base-4.5.0-py3-none-any.whl", hash = "sha256:94f3a69c32e6004cca70077f4e5c21e13ca39a281ba11a0838b4003a5f70841b"}, - {file = "cmem_plugin_base-4.5.0.tar.gz", hash = "sha256:ba9942efa09708baffe7fcbb08b28fc1a5b772825509eeb850f438802305af5f"}, -] - -[package.dependencies] -cmem-cmempy = ">=23.3.0" -python-ulid = ">=2.2.0,<3.0.0" - [[package]] name = "colorama" version = "0.4.6" @@ -307,20 +274,6 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "isodate" -version = "0.6.1" -description = "An ISO 8601 date/time/duration parser and formatter" -optional = false -python-versions = "*" -files = [ - {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, - {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, -] - -[package.dependencies] -six = "*" - [[package]] name = "jinja2" version = "3.1.3" @@ -498,14 +451,12 @@ files = [ {file = "memray-1.11.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eedea42d13b3630faa5591e298659f34e6ead06aa867050def12de6cc03e1a97"}, {file = "memray-1.11.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:9fbb2a1a82e24f0f90a9bb4ca7af6174ce91c5f3b3ce58e0b16361e989ea7cc1"}, {file = "memray-1.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6f46e00d4a10a7fb73b560e57689a68ca3661bf969e228093d20fc1313c42f0b"}, - {file = "memray-1.11.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:016a68de76fc800554fcc7dc473e48092d749b3b4302a6babd2e592a5fe8ae5e"}, {file = "memray-1.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7824202d23e3060c7a0380e1a9bb6f131f47ee29c6f30b322e844648ea3aa9da"}, {file = "memray-1.11.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5b8860e3cc7df4f7f451e043aabe60a3812f99c3e308f0c4c0e7a03d72c1563"}, {file = "memray-1.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3fc83741aedd6daa9c49ecee0a8e0048f278b6eb1ae22bdcf9b4523be7ba7106"}, {file = "memray-1.11.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:39bbf9e74c3933a84c22e047920a0f6e2d491ba943a39f4aa041f1c0422c8403"}, {file = "memray-1.11.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:0ed869e4a82722a4558da749f39d6079f2ef5e767d1399d2d090b04742e2b3f2"}, {file = "memray-1.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad1f2bb1223759e6b9755b6139087f6bcbaca1718cfed70c31aba0943542b431"}, - {file = "memray-1.11.0-cp312-cp312-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c577e81f8f7cd1418c7bfae4651d9bb3f16b72200e4b8d9b80c71aeeab64bb8"}, {file = "memray-1.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1534520c3c3e6b8234fe13c6d36bd74ab855dc19cef6e9d190a2a0b48fd2d83d"}, {file = "memray-1.11.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3dfb2c20fbbb128489f7b9f5bd2704bae6f77dba11c253cccf8eb8299697fe4"}, {file = "memray-1.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8e02e8bbe03826c5e65c2cc28760b1d0bc59f9bee6d6769c01e800b50542f5b"}, @@ -736,20 +687,6 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] -[[package]] -name = "pyparsing" -version = "3.1.2" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -optional = false -python-versions = ">=3.6.8" -files = [ - {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, - {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, -] - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - [[package]] name = "pytest" version = "7.4.4" @@ -837,41 +774,6 @@ files = [ [package.extras] cli = ["click (>=5.0)"] -[[package]] -name = "python-ulid" -version = "2.4.0.post0" -description = "Universally unique lexicographically sortable identifier" -optional = false -python-versions = ">=3.9" -files = [ - {file = "python_ulid-2.4.0.post0-py3-none-any.whl", hash = "sha256:e2c739e27e6d760136e5f411f311cdd3ec9c4c89696932fe803fa09a4dcd6ebe"}, - {file = "python_ulid-2.4.0.post0.tar.gz", hash = "sha256:45779c68b9060beb6fca72338a0620114489e1bbe274935149f14d1f776d4c43"}, -] - -[package.extras] -pydantic = ["pydantic (>=2.0)"] - -[[package]] -name = "rdflib" -version = "6.3.2" -description = "RDFLib is a Python library for working with RDF, a simple yet powerful language for representing information." -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "rdflib-6.3.2-py3-none-any.whl", hash = "sha256:36b4e74a32aa1e4fa7b8719876fb192f19ecd45ff932ea5ebbd2e417a0247e63"}, - {file = "rdflib-6.3.2.tar.gz", hash = "sha256:72af591ff704f4caacea7ecc0c5a9056b8553e0489dd4f35a9bc52dbd41522e0"}, -] - -[package.dependencies] -isodate = ">=0.6.0,<0.7.0" -pyparsing = ">=2.1.0,<4" - -[package.extras] -berkeleydb = ["berkeleydb (>=18.1.0,<19.0.0)"] -html = ["html5lib (>=1.0,<2.0)"] -lxml = ["lxml (>=4.3.0,<5.0.0)"] -networkx = ["networkx (>=2.0.0,<3.0.0)"] - [[package]] name = "requests" version = "2.31.0" @@ -893,20 +795,6 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "requests-toolbelt" -version = "1.0.0" -description = "A utility belt for advanced users of python-requests" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, - {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, -] - -[package.dependencies] -requests = ">=2.0.1,<3.0.0" - [[package]] name = "rich" version = "13.7.1" @@ -985,17 +873,6 @@ docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - [[package]] name = "textual" version = "0.55.1" @@ -1060,4 +937,4 @@ zstd = ["zstandard (>=0.18.0)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "d154f7d9ea09bd33a4d438626289797dcf6fc528471996cb46932d4c8b94686b" +content-hash = "63603e96ffbb9401838fd16ab3eba63df4815da9ef0c935ec01f8d9561d8847c" diff --git a/pyproject.toml b/pyproject.toml index f49e556..6795ccf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,28 +1,20 @@ [tool.poetry] -name = "cmem-plugin-imr" +name = "imr" version = "0.0.0" license = "Apache-2.0" description = "Inteligent Model Registry" authors = ["eccenca GmbH "] classifiers = [ "Development Status :: 4 - Beta", - "Environment :: Plugins", "Topic :: Software Development :: Libraries :: Python Modules" ] readme = "README-public.md" -keywords = [ - "eccenca Corporate Memory", "plugin" -] homepage = "https://github.com/eccenca/imr" [tool.poetry.dependencies] # if you need to change python version here, change it also in .python-version python = "^3.11" -[tool.poetry.dependencies.cmem-plugin-base] -version = "^4.3.0" -allow-prereleases = false - [tool.poetry.group.dev.dependencies] genbadge = {extras = ["coverage"], version = "^1.1.1"} mypy = "^1.2.0" diff --git a/tests/test_example.py b/tests/test_example.py index 186495a..5f4a944 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -1,76 +1,7 @@ -"""Plugin tests.""" -import io +"""Example Test""" +from imr import dummy_function -import pytest -from cmem.cmempy.workspace.projects.datasets.dataset import make_new_dataset -from cmem.cmempy.workspace.projects.project import delete_project, make_new_project -from cmem.cmempy.workspace.projects.resources.resource import ( - create_resource, - get_resource_response, -) -from cmem_plugin_imr.example_transform import Lifetime -from cmem_plugin_imr.example_workflow import DollyPlugin -from tests.utils import TestExecutionContext, needs_cmem - -PROJECT_NAME = "imr_test_project" -DATASET_NAME = "sample_dataset" -RESOURCE_NAME = "sample_dataset.txt" -DATASET_TYPE = "text" - - -@pytest.fixture() -def di_environment() -> object: - """Provide the DI build project incl. assets.""" - make_new_project(PROJECT_NAME) - make_new_dataset( - project_name=PROJECT_NAME, - dataset_name=DATASET_NAME, - dataset_type=DATASET_TYPE, - parameters={"file": RESOURCE_NAME}, - autoconfigure=False, - ) - with io.StringIO("imr plugin sample file.") as response_file: - create_resource( - project_name=PROJECT_NAME, - resource_name=RESOURCE_NAME, - file_resource=response_file, - replace=True, - ) - yield { - "project": PROJECT_NAME, - "dataset": RESOURCE_NAME, - } - delete_project(PROJECT_NAME) - - -@needs_cmem -def test_workflow_execution() -> None: - """Test plugin execution""" - entities = 100 - values = 10 - - plugin = DollyPlugin(number_of_entities=entities, number_of_values=values) - result = plugin.execute(inputs=(), context=TestExecutionContext()) - for item in result.entities: - assert len(item.values) == len(result.schema.paths) - - -def test_transform_execution_with_optional_input() -> None: - """Test Lifetime with optional input""" - result = Lifetime(start_date="2000-05-22").transform(inputs=[]) - for item in result: - assert item == "23" - - -def test_transform_execution_with_inputs() -> None: - """Test Lifetime with sequence of inputs.""" - result = Lifetime(start_date="").transform(inputs=[["2000-05-22", "2021-12-12", "1904-02-29"]]) - assert list(result) >= ["22", "0", "118"] - - -@needs_cmem -def test_integration_placeholder(di_environment: dict) -> None: - """Placeholder to write integration testcase with cmem""" - with get_resource_response(di_environment["project"], di_environment["dataset"]) as response: - assert response.text != "" +def test_dummy() -> None: + """Dummy Test""" + assert dummy_function(1) == 2 # noqa: PLR2004