From 3d5d0061c4553f63b5424571cf99d05cc155c893 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Thu, 12 Dec 2024 10:48:50 +0100 Subject: [PATCH 1/8] names: add internal id to identify the names --- .../3ba812d80559_add_internal_name_id.py | 36 +++++++++++++++++++ .../contrib/names/components.py | 24 +++++++++++++ invenio_vocabularies/contrib/names/config.py | 2 ++ .../names/jsonschemas/names/name-v1.0.0.json | 2 +- invenio_vocabularies/contrib/names/names.py | 4 ++- invenio_vocabularies/contrib/names/schema.py | 3 +- .../contrib/names/services.py | 4 +-- tests/contrib/names/test_names_service.py | 3 +- 8 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 invenio_vocabularies/alembic/3ba812d80559_add_internal_name_id.py create mode 100644 invenio_vocabularies/contrib/names/components.py diff --git a/invenio_vocabularies/alembic/3ba812d80559_add_internal_name_id.py b/invenio_vocabularies/alembic/3ba812d80559_add_internal_name_id.py new file mode 100644 index 00000000..102480e7 --- /dev/null +++ b/invenio_vocabularies/alembic/3ba812d80559_add_internal_name_id.py @@ -0,0 +1,36 @@ +# +# This file is part of Invenio. +# Copyright (C) 2024 CERN. +# +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Add internal name ID.""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "3ba812d80559" +down_revision = "55a700f897b6" +branch_labels = () +depends_on = None + + +def upgrade(): + """Upgrade database.""" + op.add_column( + "name_metadata", sa.Column("internal_id", sa.String(length=255), nullable=True) + ) + op.create_unique_constraint( + op.f("uq_name_metadata_internal_id"), "name_metadata", ["internal_id"] + ) + + +def downgrade(): + """Downgrade database.""" + op.drop_constraint( + op.f("uq_name_metadata_internal_id"), "name_metadata", type_="unique" + ) + op.drop_column("name_metadata", "internal_id") diff --git a/invenio_vocabularies/contrib/names/components.py b/invenio_vocabularies/contrib/names/components.py new file mode 100644 index 00000000..73ec07a3 --- /dev/null +++ b/invenio_vocabularies/contrib/names/components.py @@ -0,0 +1,24 @@ +# +# This file is part of Invenio. +# Copyright (C) 2024 CERN. +# +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Names service components.""" + +from invenio_records_resources.services.records.components import ServiceComponent + + +class InternalIDComponent(ServiceComponent): + """Service component for internal id field.""" + + field = "internal_id" + + def create(self, identity, data=None, record=None, **kwargs): + """Create handler.""" + setattr(record, self.field, data.pop(self.field, None)) + + def update(self, identity, data=None, record=None, **kwargs): + """Update handler.""" + setattr(record, self.field, data.pop(self.field, None)) diff --git a/invenio_vocabularies/contrib/names/config.py b/invenio_vocabularies/contrib/names/config.py index f5ddab0b..509c7c89 100644 --- a/invenio_vocabularies/contrib/names/config.py +++ b/invenio_vocabularies/contrib/names/config.py @@ -21,6 +21,7 @@ from werkzeug.local import LocalProxy from ...services.components import PIDComponent +from .components import InternalIDComponent names_schemes = LocalProxy(lambda: current_app.config["VOCABULARIES_NAMES_SCHEMES"]) @@ -67,6 +68,7 @@ class NamesSearchOptions(SearchOptions): service_components = [ # Order of components are important! + InternalIDComponent, DataComponent, PIDComponent, RelationsComponent, diff --git a/invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json b/invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json index 77a638ed..d112d306 100644 --- a/invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json +++ b/invenio_vocabularies/contrib/names/jsonschemas/names/name-v1.0.0.json @@ -65,4 +65,4 @@ "uniqueItems": true } } -} \ No newline at end of file +} diff --git a/invenio_vocabularies/contrib/names/names.py b/invenio_vocabularies/contrib/names/names.py index 791ceb8c..f01d76b1 100644 --- a/invenio_vocabularies/contrib/names/names.py +++ b/invenio_vocabularies/contrib/names/names.py @@ -13,7 +13,7 @@ from invenio_records.dumpers import SearchDumper from invenio_records.dumpers.indexedat import IndexedAtDumperExt from invenio_records.dumpers.relations import RelationDumperExt -from invenio_records.systemfields import RelationsField +from invenio_records.systemfields import ModelField, RelationsField from invenio_records_resources.factories.factory import RecordTypeFactory from invenio_records_resources.records.systemfields import ( ModelPIDField, @@ -47,10 +47,12 @@ # cannot set to nullable=False because it would fail at # service level when create({}), see records-resources. "pid": db.Column(db.String(255), unique=True), + "internal_id": db.Column(db.String(255), unique=True, nullable=True), }, schema_version="1.0.0", schema_path="local://names/name-v1.0.0.json", index_name="names-name-v2.0.0", + record_cls_attrs={"internal_id": ModelField("internal_id", dump=False)}, record_relations=name_relations, record_dumper=SearchDumper( model_fields={"pid": ("id", str)}, diff --git a/invenio_vocabularies/contrib/names/schema.py b/invenio_vocabularies/contrib/names/schema.py index 16040d09..1c39c57c 100644 --- a/invenio_vocabularies/contrib/names/schema.py +++ b/invenio_vocabularies/contrib/names/schema.py @@ -35,6 +35,7 @@ class NameSchema(BaseVocabularySchema, ModePIDFieldVocabularyMixin): so it does not inherit from it. """ + internal_id = fields.Str(allow_none=True) name = SanitizedUnicode() given_name = SanitizedUnicode() family_name = SanitizedUnicode() @@ -66,7 +67,7 @@ def validate_names(self, data, **kwargs): raise ValidationError({"family_name": messages}) @validates_schema - def validate_affiliatons(self, data, **kwargs): + def validate_affiliations(self, data, **kwargs): """Validate names.""" affiliations = data.get("affiliations", []) seen_names = set() diff --git a/invenio_vocabularies/contrib/names/services.py b/invenio_vocabularies/contrib/names/services.py index a6ab939b..7200dc1a 100644 --- a/invenio_vocabularies/contrib/names/services.py +++ b/invenio_vocabularies/contrib/names/services.py @@ -42,10 +42,10 @@ def resolve(self, identity, id_, id_type, many=False): else: results = self._read_many(identity, search_query, max_records=1) - # cant use the results_item because it returns dicts intead of records + # cant use the results_item because it returns dicts instead of records total = results.hits.total["value"] if total == 0: - # Not a PID but trated as such + # Not a PID but treated as such raise PIDDoesNotExistError(pid_type=id_type, pid_value=id_) if many: for result in results: diff --git a/tests/contrib/names/test_names_service.py b/tests/contrib/names/test_names_service.py index b8e85749..261373a9 100644 --- a/tests/contrib/names/test_names_service.py +++ b/tests/contrib/names/test_names_service.py @@ -27,6 +27,7 @@ def test_simple_flow(app, service, identity, name_full_data, example_affiliation # Create it item = service.create(identity, name_full_data) + id_ = item.id # add dereferenced values @@ -68,7 +69,7 @@ def test_simple_flow(app, service, identity, name_full_data, example_affiliation # Fail to retrieve it # - db # only the metadata is removed from the record, it is still resolvable - base_keys = {"created", "updated", "id", "links", "revision_id"} + base_keys = {"created", "updated", "id", "links", "revision_id", "internal_id"} deleted_rec = service.read(identity, id_).to_dict() assert set(deleted_rec.keys()) == base_keys # - search From 8e25dd6463cea52751bc7f17ffb4f491ffa855d1 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Thu, 12 Dec 2024 14:49:24 +0100 Subject: [PATCH 2/8] release: v6.10.0 --- CHANGES.rst | 4 ++++ invenio_vocabularies/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b899a18c..c7194887 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,10 @@ Changes ======= +Version v6.10.0 (released 2024-12-09) + +names: add internal id column to the name_metadata db + Version v6.9.0 (released 2024-12-09) - schema: added identifiers in affiliations relation diff --git a/invenio_vocabularies/__init__.py b/invenio_vocabularies/__init__.py index 27086ed3..86e7badd 100644 --- a/invenio_vocabularies/__init__.py +++ b/invenio_vocabularies/__init__.py @@ -10,6 +10,6 @@ from .ext import InvenioVocabularies -__version__ = "6.9.0" +__version__ = "6.10.0" __all__ = ("__version__", "InvenioVocabularies") From a1596f8a594d28000a886321ee931df6d31aba97 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Thu, 12 Dec 2024 18:08:49 +0100 Subject: [PATCH 3/8] names: drop unique constraint on internal id --- ...drop_unique_constraint_from_internal_id.py | 37 +++++++++++++++++++ invenio_vocabularies/contrib/names/names.py | 2 +- 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py diff --git a/invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py b/invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py new file mode 100644 index 00000000..3b7d51f8 --- /dev/null +++ b/invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py @@ -0,0 +1,37 @@ +# +# This file is part of Invenio. +# Copyright (C) 2016-2018 CERN. +# +# Invenio is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Drop unique constraint from internal id.""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "af2457652217" +down_revision = "3ba812d80559" +branch_labels = () +depends_on = None + + +def upgrade(): + """Upgrade database.""" + op.drop_constraint("uq_name_metadata_internal_id", "name_metadata", type_="unique") + op.create_index( + op.f("ix_name_metadata_internal_id"), + "name_metadata", + ["internal_id"], + unique=False, + ) + + +def downgrade(): + """Downgrade database.""" + op.drop_index(op.f("ix_name_metadata_internal_id"), table_name="name_metadata") + op.create_unique_constraint( + "uq_name_metadata_internal_id", "name_metadata", ["internal_id"] + ) diff --git a/invenio_vocabularies/contrib/names/names.py b/invenio_vocabularies/contrib/names/names.py index f01d76b1..9d71ee05 100644 --- a/invenio_vocabularies/contrib/names/names.py +++ b/invenio_vocabularies/contrib/names/names.py @@ -47,7 +47,7 @@ # cannot set to nullable=False because it would fail at # service level when create({}), see records-resources. "pid": db.Column(db.String(255), unique=True), - "internal_id": db.Column(db.String(255), unique=True, nullable=True), + "internal_id": db.Column(db.String(255), nullable=True, index=True), }, schema_version="1.0.0", schema_path="local://names/name-v1.0.0.json", From 412e70cabd47642eae1f71f034d4abfd3489f539 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Thu, 12 Dec 2024 18:09:46 +0100 Subject: [PATCH 4/8] release: v6.10.1 --- CHANGES.rst | 8 ++++++-- invenio_vocabularies/__init__.py | 2 +- ...f2457652217_drop_unique_constraint_from_internal_id.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c7194887..a74151e5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,9 +8,13 @@ Changes ======= -Version v6.10.0 (released 2024-12-09) +Version v6.10.1 (released 2024-12-12) -names: add internal id column to the name_metadata db +- names: drop unique id on the internal id + +Version v6.10.0 (released 2024-12-12) + +- names: add internal id column to the name_metadata db Version v6.9.0 (released 2024-12-09) diff --git a/invenio_vocabularies/__init__.py b/invenio_vocabularies/__init__.py index 86e7badd..80be667e 100644 --- a/invenio_vocabularies/__init__.py +++ b/invenio_vocabularies/__init__.py @@ -10,6 +10,6 @@ from .ext import InvenioVocabularies -__version__ = "6.10.0" +__version__ = "6.10.1" __all__ = ("__version__", "InvenioVocabularies") diff --git a/invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py b/invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py index 3b7d51f8..42f93e2a 100644 --- a/invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py +++ b/invenio_vocabularies/alembic/af2457652217_drop_unique_constraint_from_internal_id.py @@ -7,8 +7,8 @@ """Drop unique constraint from internal id.""" -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. From 445e49952b272e8b8f788d694d8dada0715a8228 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 12 Dec 2024 11:38:37 +0100 Subject: [PATCH 5/8] setup: bump major dependencies --- setup.cfg | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1a4b19fd..45e52437 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,10 +27,10 @@ packages = find: python_requires = >=3.7 zip_safe = False install_requires = - invenio-i18n>=2.0.0,<3.0.0 - invenio-records-resources>=6.0.0,<7.0.0 - invenio-administration>=2.0.0,<3.0.0 - invenio-jobs>=1.0.0,<2.0.0 + invenio-i18n>=3.0.0,<4.0.0 + invenio-records-resources>=7.0.0,<8.0.0 + invenio-administration>=3.0.0,<4.0.0 + invenio-jobs>=3.0.0.dev1,<4.0.0 lxml>=4.5.0 # Upper pinning pycountry due to commonmeta-py pycountry>=22.3.5,<23.0.0 @@ -49,17 +49,17 @@ sparql = SPARQLWrapper>=2.0.0 tests = pytest-black-ng>=0.4.0 - invenio-app>=1.4.0,<2.0.0 - invenio-db[postgresql,mysql]>=1.0.14,<2.0.0 + invenio-app>=2.0.0,<3.0.0 + invenio-db[postgresql,mysql]>=2.0.0,<3.0.0 pytest_httpserver>=1.0.10 - pytest-invenio>=2.1.0,<3.0.0 + pytest-invenio>=3.0.0,<4.0.0 Sphinx>=4.5 elasticsearch7 = - invenio-search[elasticsearch7]>=2.1.0,<3.0.0 + invenio-search[elasticsearch7]>=3.0.0,<4.0.0 opensearch1 = - invenio-search[opensearch1]>=2.1.0,<3.0.0 + invenio-search[opensearch1]>=3.0.0,<4.0.0 opensearch2 = - invenio-search[opensearch2]>=2.1.0,<3.0.0 + invenio-search[opensearch2]>=3.0.0,<4.0.0 # Kept for backwards compatibility: mysql = postgresql = From 03e6f64a89a0029a24335ce9a5ec4be38830af79 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 12 Dec 2024 11:38:42 +0100 Subject: [PATCH 6/8] setup: change to reusable workflows --- .github/workflows/pypi-publish.yml | 24 ++------ .github/workflows/tests-feature.yaml | 82 ---------------------------- 2 files changed, 4 insertions(+), 102 deletions(-) delete mode 100644 .github/workflows/tests-feature.yaml diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index a11ccd3c..46414e5c 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -7,23 +7,7 @@ on: jobs: build-n-publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v2 - with: - python-version: 3.7 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel babel - - name: Build package - run: | - python setup.py compile_catalog sdist bdist_wheel - - name: Publish - uses: pypa/gh-action-pypi-publish@v1.3.1 - with: - user: __token__ - - password: ${{ secrets.pypi_token }} + uses: inveniosoftware/workflows/.github/workflows/pypi-publish.yml@master + secrets: inherit + with: + babel-compile-catalog: true diff --git a/.github/workflows/tests-feature.yaml b/.github/workflows/tests-feature.yaml deleted file mode 100644 index ed404b04..00000000 --- a/.github/workflows/tests-feature.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of Invenio. -# Copyright (C) 2022 CERN. -# -# Invenio is free software; you can redistribute it and/or modify it -# under the terms of the MIT License; see LICENSE file for more details. - -name: Feature development CI - -on: - pull_request: - branches: - - "feature/**" - schedule: - # * is a special character in YAML so you have to quote this string - - cron: "0 3 * * 6" - workflow_dispatch: - inputs: - reason: - description: "Reason" - required: false - default: "Manual trigger" - -jobs: - Tests: - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: [3.8] # for efficiency test only one specific python version - requirements-level: [pypi] - cache-service: [redis] - db-service: [postgresql13] - search-service: [opensearch2,elasticsearch7] - include: - - db-service: postgresql13 - DB_EXTRAS: "postgresql" - - - search-service: opensearch2 - SEARCH_EXTRAS: "opensearch2" - - - search-service: elasticsearch7 - SEARCH_EXTRAS: "elasticsearch7" - - env: - CACHE: ${{ matrix.cache-service }} - DB: ${{ matrix.db-service }} - SEARCH: ${{ matrix.search-service }} - EXTRAS: tests,${{ matrix.SEARCH_EXTRAS }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Generate dependencies - run: | - pip install wheel requirements-builder - requirements-builder -e "$EXTRAS" --level=${{ matrix.requirements-level }} setup.py > .${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt - - - name: Cache pip - uses: actions/cache@v2 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('.${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt') }} - - - name: Install dependencies - run: | - - pip install -r .${{ matrix.requirements-level }}-${{ matrix.python-version }}-requirements.txt - pip install -r requirements-feature.txt # this file is used only when targeting a feature/* branch - pip install ".[$EXTRAS]" - pip freeze - docker --version - docker-compose --version - - - name: Run tests - run: | - ./run-tests.sh From c6b26cc09d18edc3886bf2140b0de7682a4dd769 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 12 Dec 2024 14:41:02 +0100 Subject: [PATCH 7/8] comp: make compatible to flask-sqlalchemy>=3.1 --- tests/test_alembic.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_alembic.py b/tests/test_alembic.py index 1fea19c8..20910fef 100644 --- a/tests/test_alembic.py +++ b/tests/test_alembic.py @@ -3,6 +3,7 @@ # This file is part of Invenio. # Copyright (C) 2021 TU Wien. # Copyright (C) 2021 Northwestern University. +# Copyright (C) 2024 Graz University of Technology. # # Invenio is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. @@ -46,7 +47,7 @@ def test_alembic(app, database): raise pytest.skip("Upgrades are not supported on SQLite.") # Check that this package's SQLAlchemy models have been properly registered - tables = [x.name for x in db.get_tables_for_bind()] + tables = [x for x in db.metadata.tables] assert "vocabularies_metadata" in tables assert "vocabularies_types" in tables assert "vocabularies_schemes" in tables From 4fcb870100683f69dd227cfaf53e93a785bb2dd4 Mon Sep 17 00:00:00 2001 From: Christoph Ladurner Date: Thu, 12 Dec 2024 14:41:58 +0100 Subject: [PATCH 8/8] release: v7.0.0.dev1 --- CHANGES.rst | 7 +++++++ invenio_vocabularies/__init__.py | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index a74151e5..b062d36d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,5 +1,6 @@ .. Copyright (C) 2020-2024 CERN. + Copyright (C) 2024 Graz University of Technology. Invenio-Vocabularies is free software; you can redistribute it and/or modify it under the terms of the MIT License; see LICENSE file for more @@ -8,6 +9,12 @@ Changes ======= +Version 7.0.0.dev1 (released 2024-12-12) + +- comp: make compatible to flask-sqlalchemy>=3.1 +- setup: change to reusable workflows +- setup: bump major dependencies + Version v6.10.1 (released 2024-12-12) - names: drop unique id on the internal id diff --git a/invenio_vocabularies/__init__.py b/invenio_vocabularies/__init__.py index 80be667e..dd0291a8 100644 --- a/invenio_vocabularies/__init__.py +++ b/invenio_vocabularies/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020-2024 CERN. +# Copyright (C) 2024 Graz University of Technology. # # Invenio-Vocabularies is free software; you can redistribute it and/or # modify it under the terms of the MIT License; see LICENSE file for more @@ -10,6 +11,6 @@ from .ext import InvenioVocabularies -__version__ = "6.10.1" +__version__ = "7.0.0.dev1" __all__ = ("__version__", "InvenioVocabularies")