migrator: add oauth2server token create action

migrator/tests/actions/conftest.py

@@ -29,6 +29,7 @@
     FilesObjectVersion,
 )
 from invenio_rdm_migrator.streams.models.oai import OAISet
+from invenio_rdm_migrator.streams.models.oauth import ServerClient, ServerToken
 from invenio_rdm_migrator.streams.models.pids import PersistentIdentifier
 from invenio_rdm_migrator.streams.models.records import (
     RDMDraftFile,
@@ -160,6 +161,8 @@ def database(engine):
         RDMParentMetadata,
         RDMVersionState,
         RDMParentCommunityMetadata,
+        ServerClient,
+        ServerToken,
         SessionActivity,
         User,
     ]

migrator/tests/actions/oauth/conftest.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# ZenodoRDM is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Migrator community actions tests configuration."""
+
+from pathlib import Path
+
+import pytest
+
+
+# FIXME: deduplicate from actions/communities tests
+@pytest.fixture()
+def tx_files():
+    """Transactions file paths."""
+    testdata_dir = Path(__file__).parent / "testdata"
+    assert testdata_dir.exists()
+    return {f.stem: f for f in testdata_dir.iterdir() if f.is_file()}

migrator/tests/actions/oauth/test_oauth_actions.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# ZenodoRDM is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Test OAuth actions for RDM migration."""
+
+from pathlib import Path
+
+import orjson
+import pytest
+from invenio_rdm_migrator.extract import Tx
+from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType
+from invenio_rdm_migrator.streams.actions import load
+
+from zenodo_rdm_migrator.actions.transform import OAuthServerTokenCreateAction
+
+
+@pytest.fixture()
+def create_oauth_server_token_tx(tx_files):
+    """Transaction data to create an OAuth server token.
+
+    As it would be after the extraction step.
+    """
+    datafile = Path(__file__).parent / "testdata" / "create.jsonl"
+    with open(datafile, "rb") as reader:
+        ops = [orjson.loads(line)["value"] for line in reader]
+
+    return {"tx_id": 1, "operations": ops}
+
+
+class TestOAuthServerTokenCreateAction:
+    """Create OAuth server token action tests."""
+
+    def test_matches_with_valid_data(self):
+        assert (
+            OAuthServerTokenCreateAction.matches_action(
+                Tx(
+                    id=1,
+                    operations=[
+                        {
+                            "op": OperationType.INSERT,
+                            "source": {"table": "oauth2server_client"},
+                            "after": {},
+                        },
+                        {
+                            "op": OperationType.INSERT,
+                            "source": {"table": "oauth2server_token"},
+                            "after": {},
+                        },
+                    ],
+                )
+            )
+            is True
+        )
+
+    def test_matches_with_invalid_data(self):
+        missing_client = [
+            {"op": OperationType.INSERT, "source": {"table": "another"}, "after": {}},
+            {
+                "op": OperationType.INSERT,
+                "source": {"table": "oauth2server_token"},
+                "after": {},
+            },
+        ]
+
+        missing_token = [
+            {
+                "op": OperationType.INSERT,
+                "source": {"table": "oauth2server_client"},
+                "after": {},
+            },
+            {"op": OperationType.INSERT, "source": {"table": "another"}, "after": {}},
+        ]
+
+        only_client = [
+            {"op": OperationType.INSERT, "source": {"table": "another"}, "after": {}}
+        ]
+
+        only_token = [
+            {
+                "op": OperationType.INSERT,
+                "source": {"table": "oauth2server_token"},
+                "after": {},
+            }
+        ]
+
+        wrong_op_token = [
+            {
+                "op": OperationType.INSERT,
+                "source": {"table": "oauth2server_client"},
+                "after": {},
+            },
+            {
+                "op": OperationType.UPDATE,
+                "source": {"table": "oauth2server_token"},
+                "after": {},
+            },
+        ]
+
+        wrong_op_client = [
+            {
+                "op": OperationType.UPDATE,
+                "source": {"table": "oauth2server_client"},
+                "after": {},
+            },
+            {
+                "op": OperationType.INSERT,
+                "source": {"table": "oauth2server_token"},
+                "after": {},
+            },
+        ]
+
+        extra_op = [
+            {
+                "op": OperationType.INSERT,
+                "source": {"table": "another"},
+                "after": {},
+            },
+            {
+                "op": OperationType.INSERT,
+                "source": {"table": "oauth2server_token"},
+                "after": {},
+            },
+            {"op": OperationType.INSERT, "source": {"table": "another"}, "after": {}},
+        ]
+
+        for invalid_ops in [
+            missing_client,
+            missing_token,
+            wrong_op_client,
+            wrong_op_token,
+            extra_op,
+        ]:
+            assert (
+                OAuthServerTokenCreateAction.matches_action(
+                    Tx(id=1, operations=invalid_ops)
+                )
+                is False
+            )
+
+    def test_transform_with_valid_data(self, create_oauth_server_token_tx):
+        action = OAuthServerTokenCreateAction(
+            Tx(
+                id=create_oauth_server_token_tx["tx_id"],
+                operations=create_oauth_server_token_tx["operations"],
+            )
+        )
+        assert isinstance(action.transform(), load.OAuthServerTokenCreateAction)

migrator/tests/actions/oauth/test_oauth_actions_stream.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# ZenodoRDM is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Test OAuth action stream for RDM migration."""
+
+import sqlalchemy as sa
+from invenio_rdm_migrator.streams import Stream
+from invenio_rdm_migrator.streams.models.oauth import ServerClient, ServerToken
+
+from zenodo_rdm_migrator.transform.transactions import ZenodoTxTransform
+
+
+def test_community_create_action_stream(
+    database, session, pg_tx_load, test_extract_cls, tx_files
+):
+    stream = Stream(
+        name="action",
+        extract=test_extract_cls(tx_files["create"]),
+        transform=ZenodoTxTransform(),
+        load=pg_tx_load,
+    )
+    stream.run()
+
+    assert session.scalars(sa.select(ServerClient)).one()
+    assert session.scalars(sa.select(ServerToken)).one()

migrator/tests/actions/oauth/testdata/create.jsonl

@@ -1,2 +1,2 @@
 {"topic": "zenodo-migration.public", "partition": 0, "offset": 420641, "timestamp": 1694523120495, "timestamp_type": 0, "key": {"client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_client"}, "value": {"before": null, "after": {"name": "test-incremental-token", "description": "", "website": "", "user_id": 86261, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "client_secret": "RmDfTqWnjFBM6gKc29VN0rWZPI4wi0gHBcJQYdVNLtibTK0AR1ZWbWT5oYeQ", "is_confidential": false, "is_internal": true, "_redirect_uris": null, "_default_scopes": "deposit:actions deposit:write user:email"}, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523120027, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733803496\",\"1470733804152\"]", "schema": "public", "table": "oauth2server_client", "txId": 563773535, "lsn": 1470733804152, "xmin": null}, "op": "c", "ts_ms": 1694523120317, "transaction": {"id": "563773535:1470733804152", "total_order": 1, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 135, "serialized_value_size": 772, "serialized_header_size": -1}
-{"topic": "zenodo-migration.public", "partition": 0, "offset": 420642, "timestamp": 1694523120495, "timestamp_type": 0, "key": {"id": 156666, "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_token"}, "value": {"before": null, "after": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "SlhBTjhzbXd4QTcrRjFMcTVMRHl3QlY2Rkdib0VwREY4aDhPcHo2dUt2ZnZ3OVVPa1BvRDl0L1NRZmFrdXNIU2hJR2JWc0NHZDZSVEhVT2JQcmdjS1E9PQ==", "refresh_token": null, "expires": null, "_scopes": "deposit:actions deposit:write user:email", "is_personal": true, "is_internal": false}, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523120027, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733803496\",\"1470733842408\"]", "schema": "public", "table": "oauth2server_token", "txId": 563773535, "lsn": 1470733842408, "xmin": null}, "op": "c", "ts_ms": 1694523120317, "transaction": {"id": "563773535:1470733842408", "total_order": 2, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 91, "serialized_value_size": 804, "serialized_header_size": -1}
+{"topic": "zenodo-migration.public", "partition": 0, "offset": 420642, "timestamp": 1694523120495, "timestamp_type": 0, "key": {"id": 156666, "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_token"}, "value": {"before": null, "after": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "zmkNzdnG1PXP5C3dmZqlJw==", "refresh_token": null, "expires": null, "_scopes": "deposit:actions deposit:write user:email", "is_personal": true, "is_internal": false}, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523120027, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733803496\",\"1470733842408\"]", "schema": "public", "table": "oauth2server_token", "txId": 563773535, "lsn": 1470733842408, "xmin": null}, "op": "c", "ts_ms": 1694523120317, "transaction": {"id": "563773535:1470733842408", "total_order": 2, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 91, "serialized_value_size": 804, "serialized_header_size": -1}

migrator/tests/actions/oauth/testdata/delete.jsonl

@@ -1 +1 @@
-{"topic": "zenodo-migration.public", "partition": 0, "offset": 420646, "timestamp": 1694523146547, "timestamp_type": 0, "key": {"id": 156666, "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_token"}, "value": {"before": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "SlhBTjhzbXd4QTcrRjFMcTVMRHl3QlY2Rkdib0VwREY4aDhPcHo2dUt2ZnZ3OVVPa1BvRDl0L1NRZmFrdXNIU2hJR2JWc0NHZDZSVEhVT2JQcmdjS1E9PQ==", "refresh_token": null, "expires": null, "_scopes": "", "is_personal": true, "is_internal": false}, "after": null, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523146009, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733869488\",\"1470733869488\"]", "schema": "public", "table": "oauth2server_token", "txId": 563773542, "lsn": 1470733869488, "xmin": null}, "op": "d", "ts_ms": 1694523146112, "transaction": {"id": "563773542:1470733869488", "total_order": 1, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 91, "serialized_value_size": 764, "serialized_header_size": -1}
+{"topic": "zenodo-migration.public", "partition": 0, "offset": 420646, "timestamp": 1694523146547, "timestamp_type": 0, "key": {"id": 156666, "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_token"}, "value": {"before": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "zmkNzdnG1PXP5C3dmZqlJw==", "refresh_token": null, "expires": null, "_scopes": "", "is_personal": true, "is_internal": false}, "after": null, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523146009, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733869488\",\"1470733869488\"]", "schema": "public", "table": "oauth2server_token", "txId": 563773542, "lsn": 1470733869488, "xmin": null}, "op": "d", "ts_ms": 1694523146112, "transaction": {"id": "563773542:1470733869488", "total_order": 1, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 91, "serialized_value_size": 764, "serialized_header_size": -1}

migrator/tests/actions/oauth/testdata/update.json → migrator/tests/actions/oauth/testdata/update.jsonl

@@ -1,2 +1,2 @@
 {"topic": "zenodo-migration.public", "partition": 0, "offset": 420644, "timestamp": 1694523140034, "timestamp_type": 0, "key": {"client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_client"}, "value": {"before": {"name": "test-incremental-token", "description": "", "website": "", "user_id": 86261, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "client_secret": "RmDfTqWnjFBM6gKc29VN0rWZPI4wi0gHBcJQYdVNLtibTK0AR1ZWbWT5oYeQ", "is_confidential": false, "is_internal": true, "_redirect_uris": null, "_default_scopes": "deposit:actions deposit:write user:email"}, "after": {"name": "test-incremental-token-too", "description": "", "website": "", "user_id": 86261, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "client_secret": "RmDfTqWnjFBM6gKc29VN0rWZPI4wi0gHBcJQYdVNLtibTK0AR1ZWbWT5oYeQ", "is_confidential": false, "is_internal": true, "_redirect_uris": null, "_default_scopes": "deposit:actions deposit:write user:email"}, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523139263, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733863952\",\"1470733868536\"]", "schema": "public", "table": "oauth2server_client", "txId": 563773541, "lsn": 1470733868536, "xmin": null}, "op": "u", "ts_ms": 1694523139535, "transaction": {"id": "563773541:1470733868536", "total_order": 1, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 135, "serialized_value_size": 1111, "serialized_header_size": -1}
-{"topic": "zenodo-migration.public", "partition": 0, "offset": 420645, "timestamp": 1694523140034, "timestamp_type": 0, "key": {"id": 156666, "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_token"}, "value": {"before": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "SlhBTjhzbXd4QTcrRjFMcTVMRHl3QlY2Rkdib0VwREY4aDhPcHo2dUt2ZnZ3OVVPa1BvRDl0L1NRZmFrdXNIU2hJR2JWc0NHZDZSVEhVT2JQcmdjS1E9PQ==", "refresh_token": null, "expires": null, "_scopes": "user:email deposit:actions deposit:write", "is_personal": true, "is_internal": false}, "after": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "SlhBTjhzbXd4QTcrRjFMcTVMRHl3QlY2Rkdib0VwREY4aDhPcHo2dUt2ZnZ3OVVPa1BvRDl0L1NRZmFrdXNIU2hJR2JWc0NHZDZSVEhVT2JQcmdjS1E9PQ==", "refresh_token": null, "expires": null, "_scopes": "", "is_personal": true, "is_internal": false}, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523139263, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733863952\",\"1470733868984\"]", "schema": "public", "table": "oauth2server_token", "txId": 563773541, "lsn": 1470733868984, "xmin": null}, "op": "u", "ts_ms": 1694523139536, "transaction": {"id": "563773541:1470733868984", "total_order": 2, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 91, "serialized_value_size": 1132, "serialized_header_size": -1}
+{"topic": "zenodo-migration.public", "partition": 0, "offset": 420645, "timestamp": 1694523140034, "timestamp_type": 0, "key": {"id": 156666, "__dbz__physicalTableIdentifier": "zenodo-migration.public.oauth2server_token"}, "value": {"before": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "zmkNzdnG1PXP5C3dmZqlJw==", "refresh_token": null, "expires": null, "_scopes": "user:email deposit:actions deposit:write", "is_personal": true, "is_internal": false}, "after": {"id": 156666, "client_id": "SZLrR8ApZPeBjqj7uMB1JWXavhxebu6V0mwMtvMr", "user_id": 86261, "token_type": "bearer", "access_token": "zmkNzdnG1PXP5C3dmZqlJw==", "refresh_token": null, "expires": null, "_scopes": "", "is_personal": true, "is_internal": false}, "source": {"version": "2.3.0.Final", "connector": "postgresql", "name": "zenodo-migration", "ts_ms": 1694523139263, "snapshot": "false", "db": "zenodo", "sequence": "[\"1470733863952\",\"1470733868984\"]", "schema": "public", "table": "oauth2server_token", "txId": 563773541, "lsn": 1470733868984, "xmin": null}, "op": "u", "ts_ms": 1694523139536, "transaction": {"id": "563773541:1470733868984", "total_order": 2, "data_collection_order": 1}}, "headers": [], "checksum": null, "serialized_key_size": 91, "serialized_value_size": 1132, "serialized_header_size": -1}

migrator/zenodo_rdm_migrator/actions/transform/__init__.py

@@ -14,6 +14,7 @@
 )
 from .drafts import DraftCreateAction, DraftEditAction, DraftPublishAction
 from .files import DraftFileUploadAction
+from .oauth import OAuthServerTokenCreateAction
 from .users import UserDeactivationAction, UserEditAction, UserRegistrationAction
 
 __all__ = (

migrator/zenodo_rdm_migrator/actions/transform/oauth.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 CERN.
+#
+# ZenodoRDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Invenio RDM migration oauth actions module."""
+
+from invenio_rdm_migrator.actions import TransformAction
+from invenio_rdm_migrator.load.postgresql.transactions.operations import OperationType
+from invenio_rdm_migrator.streams.actions import load
+from invenio_rdm_migrator.streams.oauth import OAuthServerTokenTransform
+from invenio_rdm_migrator.transform import IdentityTransform
+
+
+class OAuthServerTokenCreateAction(TransformAction):
+    """Zenodo to RDM OAuth server create action."""
+
+    name = "oauth-server-token-create"
+    load_cls = load.OAuthServerTokenCreateAction
+
+    @classmethod
+    def matches_action(cls, tx):
+        """Checks if the data corresponds with that required by the action."""
+        if len(tx.operations) != 2:
+            return False
+
+        rules = {
+            "oauth2server_client": OperationType.INSERT,
+            "oauth2server_token": OperationType.INSERT,
+        }
+
+        for op in tx.operations:
+            rule = rules.pop(op["source"]["table"], None)
+            if not rule or rule != op["op"]:
+                return False
+
+        return True
+
+    def _transform_data(self):
+        """Transforms the data and returns dictionary."""
+        client_src = None
+        token_src = None
+
+        if self.tx.operations[0]["source"]["table"] == "oauth2server_client":
+            client_src = self.tx.operations[0]["after"]
+            token_src = self.tx.operations[1]["after"]
+        else:  # if it matched the rules there is no other option
+            client_src = self.tx.operations[1]["after"]
+            token_src = self.tx.operations[0]["after"]
+
+        result = {
+            "tx_id": self.tx.id,
+            "client": IdentityTransform()._transform(client_src),
+            "token": OAuthServerTokenTransform()._transform(token_src),
+        }
+
+        return result

migrator/zenodo_rdm_migrator/transform/transactions.py

@@ -17,6 +17,7 @@
     DraftCreateAction,
     DraftEditAction,
     DraftFileUploadAction,
+    OAuthServerTokenCreateAction,
     UserDeactivationAction,
     UserEditAction,
     UserRegistrationAction,
@@ -33,6 +34,7 @@ class ZenodoTxTransform(BaseTxTransform):
         DraftCreateAction,
         DraftEditAction,
         DraftFileUploadAction,
+        OAuthServerTokenCreateAction,
         UserDeactivationAction,
         UserEditAction,
         UserRegistrationAction,