Metadata form versioning - 3 (#1648)

### Feature or Bugfix
<!-- please choose -->
- Feature


### Detail

View:
- [ ] Display version on "attached form" card
- [ ] Display version in attached form list
- [ ] Show number of entities attached (MF -> Fields)
- [ ] Delete confirmation PopUp
- [ ] Enforcement tab temporary hidden

Edit:

- [ ] No editing if there are attached entities
- [ ] Editing of attached MF
- [ ] If new version is available, editing of MF automatically shows new
version

Other:
- [ ] Notifications for entity owners, if new version of MF is released

Left ToDo:
#1621 (comment)

### Relates
- #1621 

### Security
Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).

- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
  - Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
  - Is injection prevented by parametrizing queries?
  - Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
  - Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
  - Do you use a standard proven implementations?
  - Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
  - Have you used the least-privilege principle? How?


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Sofia Sazonova <[email protected]>

backend/dataall/modules/metadata_forms/api/input_types.py

@@ -71,6 +71,7 @@
     arguments=[
         gql.Field(name='entityType', type=gql.NonNullableType(gql.String)),
         gql.Field(name='entityUri', type=gql.NonNullableType(gql.String)),
+        gql.Field(name='attachedUri', type=gql.String),
         gql.Field(name='fields', type=gql.ArrayType(gql.Ref('NewAttachedMetadataFormFieldInput'))),
     ],
 )

backend/dataall/modules/metadata_forms/api/queries.py

@@ -6,6 +6,7 @@
     get_attached_metadata_form,
     list_attached_forms,
     get_entity_metadata_form_permissions,
+    list_metadata_form_versions,
 )
 
 listUserMetadataForms = gql.QueryField(
@@ -32,6 +33,14 @@
     test_scope='MetadataForm',
 )
 
+listMetadataFormVersions = gql.QueryField(
+    name='listMetadataFormVersions',
+    args=[gql.Argument('uri', gql.NonNullableType(gql.String))],
+    type=gql.ArrayType(gql.Ref('MetadataFormVersion')),
+    resolver=list_metadata_form_versions,
+    test_scope='MetadataForm',
+)
+
 listAttachedMetadataForms = gql.QueryField(
     name='listAttachedMetadataForms',
     args=[gql.Argument('filter', gql.Ref('AttachedMetadataFormFilter'))],

backend/dataall/modules/metadata_forms/api/resolvers.py

@@ -21,7 +21,7 @@ def create_metadata_form_version(context: Context, source, formUri, copyVersion)
 
 
 def create_attached_metadata_form(context: Context, source, formUri, input):
-    return AttachedMetadataFormService.create_attached_metadata_form(uri=formUri, data=input)
+    return AttachedMetadataFormService.create_or_update_attached_metadata_form(uri=formUri, data=input)
 
 
 def delete_metadata_form(context: Context, source, formUri):
@@ -48,6 +48,10 @@ def get_home_entity_name(context: Context, source: MetadataForm):
     return MetadataFormService.get_home_entity_name(metadata_form=source)
 
 
+def get_entity_name(context: Context, source: AttachedMetadataForm):
+    return MetadataFormService.get_entity_name(attached_metadata_form=source)
+
+
 def get_metadata_form(context: Context, source, uri):
     return MetadataFormService.get_metadata_form_by_uri(uri=uri)
 
@@ -106,3 +110,7 @@ def resolve_metadata_form_field(context: Context, source: AttachedMetadataFormFi
 
 def get_entity_metadata_form_permissions(context: Context, source, entityUri):
     return MetadataFormService.get_mf_permissions(entityUri=entityUri)
+
+
+def list_metadata_form_versions(context: Context, source, uri):
+    return MetadataFormService.list_metadata_form_versions(uri=uri)

backend/dataall/modules/metadata_forms/api/types.py

@@ -1,6 +1,7 @@
 from dataall.base.api import gql
 from dataall.modules.metadata_forms.api.resolvers import (
     get_home_entity_name,
+    get_entity_name,
     get_form_fields,
     get_fields_glossary_node_name,
     get_user_role,
@@ -34,6 +35,7 @@
     ],
 )
 
+
 MetadataFormField = gql.ObjectType(
     name='MetadataFormField',
     fields=[
@@ -81,13 +83,24 @@
     ],
 )
 
+MetadataFormVersion = gql.ObjectType(
+    name='MetadataFormVersion',
+    fields=[
+        gql.Field(name='metadataFormUri', type=gql.ID),
+        gql.Field(name='version', type=gql.Integer),
+        gql.Field(name='attached_forms', type=gql.Integer),
+    ],
+)
+
 AttachedMetadataForm = gql.ObjectType(
     name='AttachedMetadataForm',
     fields=[
         gql.Field(name='uri', type=gql.ID),
         gql.Field(name='metadataForm', type=gql.Ref('MetadataForm'), resolver=resolve_metadata_form),
+        gql.Field(name='version', type=gql.Integer),
         gql.Field(name='entityUri', type=gql.String),
         gql.Field(name='entityType', type=gql.String),
+        gql.Field(name='entityName', type=gql.String, resolver=get_entity_name),
         gql.Field(
             name='fields', type=gql.ArrayType(gql.Ref('AttachedMetadataFormField')), resolver=get_attached_form_fields
         ),

backend/dataall/modules/metadata_forms/db/enums.py

@@ -20,7 +20,8 @@ class MetadataFormEntityTypes(GraphQLEnumMapper):
     OrganizationTeams = 'Organization Team'
     Environments = 'Environment'
     EnvironmentTeams = 'Environment Team'
-    Datasets = 'Dataset'
+    S3Datasets = 'S3-Dataset'
+    RDDatasets = 'Redshift-Dataset'
     Worksheets = 'Worksheets'
     Dashboards = 'Dashboard'
     ConsumptionRoles = 'Consumption Role'

backend/dataall/modules/metadata_forms/db/metadata_form_repository.py

@@ -288,13 +288,16 @@ def query_attached_metadata_forms(session, is_da_admin, groups, user_envs_uris,
         # The c confuses a lot of people, SQLAlchemy uses this unfortunately odd name
         # as a container for columns in table objects.
         query = session.query(AttachedMetadataForm).join(all_mfs, AttachedMetadataForm.metadataFormUri == all_mfs.c.uri)
-        if filter and filter.get('entityType'):
-            query = query.filter(AttachedMetadataForm.entityType == filter.get('entityType'))
-        if filter and filter.get('entityUri'):
-            query = query.filter(AttachedMetadataForm.entityUri == filter.get('entityUri'))
-        if filter and filter.get('metadataFormUri'):
-            query = query.filter(AttachedMetadataForm.metadataFormUri == filter.get('metadataFormUri'))
-        return query
+        if filter:
+            if filter.get('entityType'):
+                query = query.filter(AttachedMetadataForm.entityType == filter.get('entityType'))
+            if filter.get('entityUri'):
+                query = query.filter(AttachedMetadataForm.entityUri == filter.get('entityUri'))
+            if filter.get('metadataFormUri'):
+                query = query.filter(AttachedMetadataForm.metadataFormUri == filter.get('metadataFormUri'))
+            if filter.get('version'):
+                query = query.filter(AttachedMetadataForm.version == filter.get('version'))
+        return query.order_by(all_mfs.c.name)
 
     @staticmethod
     def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType):
@@ -303,11 +306,28 @@ def query_all_attached_metadata_forms_for_entity(session, entityUri, entityType)
         )
 
     @staticmethod
-    def get_metadata_form_versions(session, uri):
+    def get_metadata_form_versions_numbers(session, uri):
         versions = (
             session.query(MetadataFormVersion)
             .filter(MetadataFormVersion.metadataFormUri == uri)
             .order_by(MetadataFormVersion.version.desc())
             .all()
         )
         return [v.version for v in versions]
+
+    @staticmethod
+    def get_metadata_form_versions(session, uri):
+        versions = (
+            session.query(MetadataFormVersion)
+            .filter(MetadataFormVersion.metadataFormUri == uri)
+            .order_by(MetadataFormVersion.version.desc())
+            .all()
+        )
+        return versions
+
+    @staticmethod
+    def get_all_attached_metadata_forms(session, mf_uri, version=None):
+        all_attached = session.query(AttachedMetadataForm).filter(AttachedMetadataForm.metadataFormUri == mf_uri)
+        if version:
+            all_attached = all_attached.filter(AttachedMetadataForm.version == version)
+        return all_attached.all()

backend/dataall/modules/metadata_forms/services/attached_metadata_form_service.py

@@ -36,6 +36,19 @@ class AttachedMetadataFormService:
     def _get_entity_uri(session, data):
         return data.get('entityUri')
 
+    @staticmethod
+    @ResourcePolicyService.has_resource_permission(
+        ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'
+    )
+    def create_or_update_attached_metadata_form(uri, data):
+        new_form = AttachedMetadataFormService.create_attached_metadata_form(uri=uri, data=data)
+        if data.get('attachedUri'):
+            with get_context().db_engine.scoped_session() as session:
+                existingAMF = MetadataFormRepository.get_attached_metadata_form(session, data.get('attachedUri'))
+                if existingAMF and new_form:
+                    session.delete(existingAMF)
+        return new_form
+
     @staticmethod
     @ResourcePolicyService.has_resource_permission(
         ATTACH_METADATA_FORM, parent_resource=_get_entity_uri, param_name='data'

backend/dataall/modules/metadata_forms/services/metadata_form_access_service.py

@@ -68,7 +68,7 @@ def _target_org_uri_getter(entityType, entityUri):
         elif entityType == MetadataFormEntityTypes.Environments.value:
             with get_context().db_engine.scoped_session() as session:
                 return [EnvironmentRepository.get_environment_by_uri(session, entityUri).organizationUri]
-        elif entityType == MetadataFormEntityTypes.Datasets.value:
+        elif entityType in [MetadataFormEntityTypes.S3Datasets.value, MetadataFormEntityTypes.RDDatasets.value]:
             with get_context().db_engine.scoped_session() as session:
                 return [DatasetBaseRepository.get_dataset_by_uri(session, entityUri).organizationUri]
         else:
@@ -83,7 +83,7 @@ def _target_env_uri_getter(entityType, entityUri):
             return None
         elif entityType == MetadataFormEntityTypes.Environments.value:
             return [entityUri]
-        elif entityType == MetadataFormEntityTypes.Datasets.value:
+        elif entityType in [MetadataFormEntityTypes.S3Datasets.value, MetadataFormEntityTypes.RDDatasets.value]:
             with get_context().db_engine.scoped_session() as session:
                 return [DatasetBaseRepository.get_dataset_by_uri(session, entityUri).environmentUri]
         else:

backend/dataall/modules/metadata_forms/services/metadata_form_service.py

@@ -6,9 +6,11 @@
 from dataall.core.permissions.db.resource_policy.resource_policy_repositories import ResourcePolicyRepository
 from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
 from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService
+from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository
 from dataall.modules.metadata_forms.db.enums import (
     MetadataFormVisibility,
     MetadataFormFieldType,
+    MetadataFormEntityTypes,
 )
 from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository
 from dataall.modules.metadata_forms.db.metadata_form_repository import MetadataFormRepository
@@ -21,6 +23,7 @@
     CREATE_METADATA_FORM,
     ALL_METADATA_FORMS_ENTITY_PERMISSIONS,
 )
+from dataall.modules.notifications.db.notification_repositories import NotificationRepository
 
 
 class MetadataFormParamValidationService:
@@ -129,7 +132,7 @@ def get_metadata_form_by_uri(uri):
         with get_context().db_engine.scoped_session() as session:
             mf = MetadataFormRepository.get_metadata_form(session, uri)
             if mf:
-                mf.versions = MetadataFormRepository.get_metadata_form_versions(session, uri)
+                mf.versions = MetadataFormRepository.get_metadata_form_versions_numbers(session, uri)
             return mf
 
     # toDo: deletion logic
@@ -295,25 +298,81 @@ def get_mf_permissions(entityUri):
     @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def create_metadata_form_version(uri, copyVersion):
         with get_context().db_engine.scoped_session() as session:
+            mf = MetadataFormService.get_metadata_form_by_uri(uri)
             new_version = MetadataFormRepository.create_metadata_form_version_next(session, uri)
             if copyVersion:
                 mf_fields = MetadataFormRepository.get_metadata_form_fields(session, uri, copyVersion)
                 for field in mf_fields:
                     new_field = MetadataFormRepository.create_metadata_form_field(
                         session, uri, field.__dict__, new_version.version
                     )
+
+            all_attached = MetadataFormRepository.get_all_attached_metadata_forms(session, uri)
+            for attached in all_attached:
+                owner = MetadataFormService.get_entity_owner(attached)
+                if owner:
+                    NotificationRepository.create_notification(
+                        session,
+                        recipient=owner,
+                        target_uri=f'{attached.entityUri}|{attached.entityType}',
+                        message=f'New version {new_version.version} is available for metadata form "{mf.name}" for {attached.entityType} {attached.entityUri}',
+                        notification_type='METADATA_FORM_UPDATE',
+                    )
         return new_version.version
 
     @staticmethod
     @TenantPolicyService.has_tenant_permission(MANAGE_METADATA_FORMS)
     @MetadataFormAccessService.can_perform(UPDATE_METADATA_FORM_FIELD)
     def delete_metadata_form_version(uri, version):
         with get_context().db_engine.scoped_session() as session:
-            all_versions = MetadataFormRepository.get_metadata_form_versions(session, uri)
+            all_versions = MetadataFormRepository.get_metadata_form_versions_numbers(session, uri)
             if len(all_versions) == 1:
                 raise UnauthorizedOperation(
                     action='Delete version', message='Cannot delete the only version of the form'
                 )
             mf = MetadataFormRepository.get_metadata_form_version(session, uri, version)
             session.delete(mf)
             return MetadataFormRepository.get_metadata_form_version_number_latest(session, uri)
+
+    @staticmethod
+    def list_metadata_form_versions(uri):
+        with get_context().db_engine.scoped_session() as session:
+            all_versions = MetadataFormRepository.get_metadata_form_versions(session, uri)
+            for v in all_versions:
+                v.attached_forms = len(MetadataFormRepository.get_all_attached_metadata_forms(session, uri, v.version))
+            return all_versions
+
+    @staticmethod
+    def resolve_attached_entity(attached_metadata_form):
+        with get_context().db_engine.scoped_session() as session:
+            if attached_metadata_form.entityType == MetadataFormEntityTypes.Organizations.value:
+                return OrganizationRepository.get_organization_by_uri(session, attached_metadata_form.entityUri)
+            elif attached_metadata_form.entityType == MetadataFormEntityTypes.Environments.value:
+                return EnvironmentRepository.get_environment_by_uri(session, attached_metadata_form.entityUri)
+            elif attached_metadata_form.entityType in [
+                MetadataFormEntityTypes.S3Datasets.value,
+                MetadataFormEntityTypes.RDDatasets.value,
+            ]:
+                return DatasetBaseRepository.get_dataset_by_uri(session, attached_metadata_form.entityUri)
+            else:
+                return None
+
+    @staticmethod
+    def get_entity_name(attached_metadata_form):
+        entity = MetadataFormService.resolve_attached_entity(attached_metadata_form)
+        return entity.name if entity else 'Not Found'
+
+    @staticmethod
+    def get_entity_owner(attached_metadata_form):
+        entity = MetadataFormService.resolve_attached_entity(attached_metadata_form)
+        if entity:
+            if attached_metadata_form.entityType == MetadataFormEntityTypes.Organizations.value:
+                return entity.SamlGroupName
+            elif attached_metadata_form.entityType == MetadataFormEntityTypes.Environments.value:
+                return entity.SamlGroupName
+            elif attached_metadata_form.entityType in [
+                MetadataFormEntityTypes.S3Datasets.value,
+                MetadataFormEntityTypes.RDDatasets.value,
+            ]:
+                return entity.SamlAdminGroupName
+        return None

backend/migrations/versions/b21f86882012_mf_attached_entity_type.py

@@ -0,0 +1,49 @@
+"""mf_attached_entity_type
+
+Revision ID: b21f86882012
+Revises: 5a798acc6282
+Create Date: 2024-10-22 15:50:42.652910
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from dataall.modules.metadata_forms.db.metadata_form_models import AttachedMetadataForm
+
+# revision identifiers, used by Alembic.
+revision = 'b21f86882012'
+down_revision = '5a798acc6282'
+branch_labels = None
+depends_on = None
+
+
+def get_session():
+    bind = op.get_bind()
+    session = orm.Session(bind=bind)
+    return session
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    session = get_session()
+    print('Rename entityType from Dataset to S3-Dataset for attached metadataform entries')
+    all_amf = session.query(AttachedMetadataForm).all()
+    for amf in all_amf:
+        if amf.entityType == 'Dataset':
+            amf.entityType = 'S3-Dataset'
+    session.commit()
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    session = get_session()
+    print('Rename entityType from S3-Dataset to Dataset for attached metadataform entries')
+    all_amf = session.query(AttachedMetadataForm).all()
+    for amf in all_amf:
+        if amf.entityType == 'S3-Dataset':
+            amf.entityType = 'Dataset'
+    session.commit()
+    # ### end Alembic commands ###