diff --git a/CodeListLibrary_project/clinicalcode/admin.py b/CodeListLibrary_project/clinicalcode/admin.py index 8939b22f2..9c95ac92a 100644 --- a/CodeListLibrary_project/clinicalcode/admin.py +++ b/CodeListLibrary_project/clinicalcode/admin.py @@ -7,10 +7,15 @@ from .models.EntityClass import EntityClass from .models.GenericEntity import GenericEntity from .models.Template import Template +from .models.OntologyTag import OntologyTag from .models.DMD_CODES import DMD_CODES from .forms.TemplateForm import TemplateAdminForm from .forms.EntityClassForm import EntityAdminForm +@admin.register(OntologyTag) +class OntologyTag(admin.ModelAdmin): + list_display = ['id', 'name', 'type_id', 'atlas_id'] + @admin.register(CodingSystemFilter) class CodingSystemFilterAdmin(admin.ModelAdmin): diff --git a/CodeListLibrary_project/clinicalcode/api/urls.py b/CodeListLibrary_project/clinicalcode/api/urls.py index 538a809c0..e6bdaccaf 100644 --- a/CodeListLibrary_project/clinicalcode/api/urls.py +++ b/CodeListLibrary_project/clinicalcode/api/urls.py @@ -7,7 +7,11 @@ from drf_yasg.views import get_schema_view from drf_yasg import openapi -from .views import Concept, GenericEntity, Template, DataSource, Tag, Collection +from .views import ( + Concept, GenericEntity, + Template, DataSource, + Tag, Collection, Ontology +) """ Router Use the default REST API router to access the API details explicitly. These paths will @@ -135,19 +139,30 @@ def get_schema(self, request=None, public=False): # Tags url(r'^tags/$', - Tag.get_tags, + Tag.get_tags, name='tag_list'), url(r'^tags/(?P\d+)/detail/$', - Tag.get_tag_detail, + Tag.get_tag_detail, name='tag_list_by_id'), # Collections url(r'^collections/$', - Collection.get_collections, + Collection.get_collections, name='collection_list'), url(r'^collections/(?P\d+)/detail/$', - Collection.get_collection_detail, - name='collections_list_by_id'), + Collection.get_collection_detail, + name='collections_list_by_id'), + + # Ontology + url(r'^ontology/$', + Ontology.get_ontologies, + name='ontology_list'), + url(r'^ontology/type/(?P\d+)/$', + Ontology.get_ontology_detail, + name='ontology_list_by_type'), + url(r'^ontology/node/(?P\d+)/$', + Ontology.get_ontology_node, + name='ontology_node_by_id'), ] """ Create/Update urls """ diff --git a/CodeListLibrary_project/clinicalcode/api/views/GenericEntity.py b/CodeListLibrary_project/clinicalcode/api/views/GenericEntity.py index a01523d23..bb9b3cb4e 100644 --- a/CodeListLibrary_project/clinicalcode/api/views/GenericEntity.py +++ b/CodeListLibrary_project/clinicalcode/api/views/GenericEntity.py @@ -318,7 +318,7 @@ def get_entity_detail(request, phenotype_id, version_id=None, field=None): historical_entity, user_authed, target_field=field, - return_data=True + return_data=False ) if field == 'codes': diff --git a/CodeListLibrary_project/clinicalcode/api/views/Ontology.py b/CodeListLibrary_project/clinicalcode/api/views/Ontology.py new file mode 100644 index 000000000..8e8988633 --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/api/views/Ontology.py @@ -0,0 +1,81 @@ +from rest_framework.decorators import (api_view, permission_classes) +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.response import Response +from rest_framework import status +from django.db.models import F + +from ...entity_utils import api_utils +from ...entity_utils import gen_utils +from ...entity_utils import constants + +from ...models.OntologyTag import OntologyTag + +@api_view(['GET']) +@permission_classes([IsAuthenticatedOrReadOnly]) +def get_ontologies(request): + """ + Get all ontology categories and their root nodes, incl. associated data + """ + result = OntologyTag.get_groups([x.value for x in constants.ONTOLOGY_TYPES], default=[]) + return Response( + data=list(result), + status=status.HTTP_200_OK + ) + +@api_view(['GET']) +@permission_classes([IsAuthenticatedOrReadOnly]) +def get_ontology_detail(request, ontology_id): + """ + Get specified ontology group detail by the ontology_id type, including associated + data e.g. root nodes, children etc + """ + ontology_id = gen_utils.parse_int(ontology_id, default=None) + if not isinstance(ontology_id, int): + return Response( + data={ + 'message': 'Invalid ontology id, expected valid integer' + }, + content_type='json', + status=status.HTTP_400_BAD_REQUEST + ) + + result = OntologyTag.get_group_data(ontology_id, default=None) + if not isinstance(result, dict): + return Response( + data={ + 'message': f'Ontology of id {ontology_id} does not exist' + }, + content_type='json', + status=status.HTTP_404_NOT_FOUND + ) + + return Response( + data=result, + status=status.HTTP_200_OK + ) + +@api_view(['GET']) +@permission_classes([IsAuthenticatedOrReadOnly]) +def get_ontology_node(request, node_id): + """ + Get the element details of the specified ontology element by its + node_id including associated data + + e.g. tree-related information + + """ + node_id = gen_utils.parse_int(node_id, default=None) + if not isinstance(node_id, int): + return Response( + data={ + 'message': 'Invalid node id, expected valid integer' + }, + content_type='json', + status=status.HTTP_400_BAD_REQUEST + ) + + result = OntologyTag.get_node_data(node_id, default=None) + return Response( + data=result, + status=status.HTTP_200_OK + ) diff --git a/CodeListLibrary_project/clinicalcode/entity_utils/api_utils.py b/CodeListLibrary_project/clinicalcode/entity_utils/api_utils.py index 26ab7abf6..8e41d4404 100644 --- a/CodeListLibrary_project/clinicalcode/entity_utils/api_utils.py +++ b/CodeListLibrary_project/clinicalcode/entity_utils/api_utils.py @@ -385,13 +385,14 @@ def get_entity_detail_from_layout( validation = template_utils.get_field_item( constants.metadata, field, 'validation', {} ) - is_source = validation.get('source') + is_source = validation.get('source') if is_source: - value = template_utils.get_metadata_value_from_source( - entity, field, default=None - ) - if value is None: + if template_utils.is_metadata(entity, field=field): + value = template_utils.get_metadata_value_from_source( + entity, field, default=None + ) + else: value = template_utils.get_entity_field(entity, field) result[field] = value diff --git a/CodeListLibrary_project/clinicalcode/entity_utils/concept_utils.py b/CodeListLibrary_project/clinicalcode/entity_utils/concept_utils.py index e89b78e31..4e7097148 100644 --- a/CodeListLibrary_project/clinicalcode/entity_utils/concept_utils.py +++ b/CodeListLibrary_project/clinicalcode/entity_utils/concept_utils.py @@ -1,3 +1,4 @@ + from django.db import connection from django.db.models import ForeignKey from django.http.request import HttpRequest @@ -745,8 +746,8 @@ def get_clinical_concept_data(concept_id, concept_history_id, include_reviewed_c aggregate_component_codes=False, include_component_codes=True, include_attributes=False, strippable_fields=None, remove_userdata=False, hide_user_details=False, - derive_access_from=None, format_for_api=False, - include_source_data=False): + derive_access_from=None, requested_entity_id=None, + format_for_api=False, include_source_data=False): """ [!] Note: This method ignores permissions to derive data - it should only be called from a a method that has previously considered accessibility @@ -878,21 +879,21 @@ def get_clinical_concept_data(concept_id, concept_history_id, include_reviewed_c concept_data['code_attribute_header'] = attribute_headers # Set phenotype owner - phenotype_owner = concept.phenotype_owner + phenotype_owner = concept.phenotype_owner if phenotype_owner: concept_data['phenotype_owner'] = phenotype_owner.id - entity_from_concept = GenericEntity.history.filter( - id=phenotype_owner.id - ) - template_data = entity_from_concept.values_list('template_data',flat=True) - #Iterate through the concept_information to find the phenotype owner history - for index, value in enumerate(template_data): - history_ids_concept = [j['concept_version_id'] for j in value['concept_information']] - if concept_history_id in history_ids_concept: - concept_data['phenotype_owner_history_id'] = entity_from_concept.values_list('history_id',flat=True)[index] - + latest_version_id = permission_utils.get_latest_owner_version_from_concept( + phenotype_id=phenotype_owner.id, + concept_id=historical_concept.id, + concept_version_id=historical_concept.history_id + ) + if latest_version_id is not None: + concept_data['phenotype_owner_history_id'] = latest_version_id + + # Set the requested entity id + concept_data['requested_entity_id'] = requested_entity_id # Set base if not format_for_api: diff --git a/CodeListLibrary_project/clinicalcode/entity_utils/constants.py b/CodeListLibrary_project/clinicalcode/entity_utils/constants.py index 6c8c11b3c..aab6b1039 100644 --- a/CodeListLibrary_project/clinicalcode/entity_utils/constants.py +++ b/CodeListLibrary_project/clinicalcode/entity_utils/constants.py @@ -125,6 +125,26 @@ class FORM_METHODS(int, enum.Enum, metaclass=IterableMeta): CREATE = 1 UPDATE = 2 +class ONTOLOGY_TYPES(int, enum.Enum, metaclass=IterableMeta): + """ + Defines the ontology internal type id, + which describes the ontology type + + """ + CLINICAL_DISEASE = 0 + CLINICAL_DOMAIN = 1 + CLINICAL_FUNCTIONAL_ANATOMY = 2 + +""" + Used to define the labels for each + known ontology type + +""" +ONTOLOGY_LABELS = { + ONTOLOGY_TYPES.CLINICAL_DOMAIN: 'Clinical Domain', + ONTOLOGY_TYPES.CLINICAL_DISEASE: 'Clinical Disease Category (ICD-10)', + ONTOLOGY_TYPES.CLINICAL_FUNCTIONAL_ANATOMY: 'Functional Anatomy', +} """ The excepted X-Requested-With header if a fetch request is made @@ -476,7 +496,7 @@ class FORM_METHODS(int, enum.Enum, metaclass=IterableMeta): "citation_requirements": { "title": "Citation Requirements", "description": "A request for how this phenotype is referenced if used in other work.", - "field_type": "textarea_markdown", + "field_type": "citation_requirements", "active": True, "validation": { "type": "string", @@ -699,6 +719,12 @@ class FORM_METHODS(int, enum.Enum, metaclass=IterableMeta): 'output_type': 'list_of_inputboxes' }, + 'citation_requirements': { + 'data_type': 'string', + 'input_type': 'markdown', + 'output_type': 'citation_requirements' + }, + 'enum': { 'data_type': 'int', 'input_type': 'dropdown-list', @@ -712,6 +738,11 @@ class FORM_METHODS(int, enum.Enum, metaclass=IterableMeta): 'apply_badge_style': True }, + 'ontology': { + 'input_type': 'generic/ontology', + 'output_type': 'generic/ontology' + }, + 'enum_radio_badge': { 'data_type': 'int', 'input_type': 'radiobutton', diff --git a/CodeListLibrary_project/clinicalcode/entity_utils/create_utils.py b/CodeListLibrary_project/clinicalcode/entity_utils/create_utils.py index a0164cd5a..005dd78f3 100644 --- a/CodeListLibrary_project/clinicalcode/entity_utils/create_utils.py +++ b/CodeListLibrary_project/clinicalcode/entity_utils/create_utils.py @@ -1,7 +1,7 @@ +from django.db import transaction, IntegrityError from django.apps import apps from django.db.models import Q from django.utils.timezone import make_aware -from django.db import transaction, IntegrityError from datetime import datetime from ..models.EntityClass import EntityClass @@ -12,8 +12,10 @@ from ..models.ConceptCodeAttribute import ConceptCodeAttribute from ..models.Component import Component from ..models.CodeList import CodeList +from ..models.OntologyTag import OntologyTag from ..models.Code import Code from ..models.Tag import Tag + from . import gen_utils from . import model_utils from . import permission_utils @@ -81,6 +83,7 @@ def get_template_creation_data(request, entity, layout, field, default=None): item['concept_id'], item['concept_version_id'], aggregate_component_codes=False, + requested_entity_id=entity.id, derive_access_from=request, include_source_data=True, include_attributes=True @@ -90,7 +93,12 @@ def get_template_creation_data(request, entity, layout, field, default=None): values.append(value) return values - + elif field_type == 'int_array': + source_info = validation.get('source') + tree_models = source_info.get('trees') if isinstance(source_info, dict) else None + if isinstance(tree_models, list): + return OntologyTag.get_creation_data(node_ids=data, type_ids=tree_models, default=default) + if template_utils.is_metadata(entity, field): return template_utils.get_metadata_value_from_source(entity, field, default=default) @@ -132,32 +140,50 @@ def try_validate_sourced_value(field, template, data, default=None, request=None validation = template_utils.try_get_content(template, 'validation') if validation: if 'source' in validation: - try: - source_info = validation.get('source') - model = apps.get_model(app_label='clinicalcode', model_name=source_info.get('table')) - - if isinstance(data, list): - query = { - 'pk__in': data - } + source_info = validation.get('source') or { } + model_name = source_info.get('table') + tree_models = source_info.get('trees') + + if isinstance(tree_models, list): + try: + model = apps.get_model(app_label='clinicalcode', model_name='OntologyTag') + queryset = model.objects.filter(pk__in=data, type_id__in=tree_models) + queryset = list(queryset.values_list('id', flat=True)) + + if isinstance(data, list): + return queryset if len(queryset) > 0 else default + except: + return default else: - query = { - 'pk': data - } - - if 'filter' in source_info: - filter_query = template_utils.try_get_filter_query(field, source_info.get('filter'), request=request) - query = {**query, **filter_query} - - queryset = model.objects.filter(Q(**query)) - queryset = list(queryset.values_list('id', flat=True)) - - if isinstance(data, list): - return queryset if len(queryset) > 0 else default - else: - return queryset[0] if len(queryset) > 0 else default - except: - return default + return default + elif isinstance(model_name, str): + try: + source_info = validation.get('source') + model = apps.get_model(app_label='clinicalcode', model_name=model_name) + + if isinstance(data, list): + query = { + 'pk__in': data + } + else: + query = { + 'pk': data + } + + if 'filter' in source_info: + filter_query = template_utils.try_get_filter_query(field, source_info.get('filter'), request=request) + query = {**query, **filter_query} + + queryset = model.objects.filter(Q(**query)) + queryset = list(queryset.values_list('id', flat=True)) + + if isinstance(data, list): + return queryset if len(queryset) > 0 else default + else: + return queryset[0] if len(queryset) > 0 else default + except: + return default + return default elif 'options' in validation: options = validation['options'] @@ -172,7 +198,7 @@ def try_validate_sourced_value(field, template, data, default=None, request=None data = str(data) if data in options: return data - + return default def validate_form_method(form_method, errors=[], default=None): @@ -572,6 +598,24 @@ def validate_metadata_value(request, field, value, errors=[]): field_value = gen_utils.try_value_as_type(value, field_type, validation) return field_value, True +def is_computed_template_field(field, form_template): + """ + Checks whether a field is considered a computed field within its template + """ + field_data = template_utils.get_layout_field(form_template, field) + if field_data is None: + return False + + validation = template_utils.try_get_content(field_data, 'validation') + if validation is None: + return False + + field_computed = template_utils.try_get_content(validation, 'computed') + if field_computed is not None: + return True + + return False + def validate_template_value(request, field, form_template, value, errors=[]): """ Validates the form's field value against the entity template @@ -660,9 +704,13 @@ def validate_entity_form(request, content, errors=[], method=None): continue top_level_data[field] = field_value elif validate_template_field(form_template, field): + if is_computed_template_field(field, form_template): + continue + field_value, validated = validate_template_value(request, field, form_template, value, errors) if not validated or field_value is None: continue + template_data[field] = field_value try_add_computed_fields(field, form_data, form_template, template_data) diff --git a/CodeListLibrary_project/clinicalcode/entity_utils/gen_utils.py b/CodeListLibrary_project/clinicalcode/entity_utils/gen_utils.py index dbc52f5d1..d259860f1 100644 --- a/CodeListLibrary_project/clinicalcode/entity_utils/gen_utils.py +++ b/CodeListLibrary_project/clinicalcode/entity_utils/gen_utils.py @@ -224,12 +224,15 @@ def parse_int(value, default=0): """ Attempts to parse an int from a value, if it fails to do so, returns the default value """ + if isinstance(value, int): + return value + if value is None: return default - + try: value = int(value) - except ValueError: + except: return default else: return value diff --git a/CodeListLibrary_project/clinicalcode/entity_utils/permission_utils.py b/CodeListLibrary_project/clinicalcode/entity_utils/permission_utils.py index d3a0e5efa..88ba62fc8 100644 --- a/CodeListLibrary_project/clinicalcode/entity_utils/permission_utils.py +++ b/CodeListLibrary_project/clinicalcode/entity_utils/permission_utils.py @@ -7,6 +7,7 @@ from functools import wraps from ..models.Concept import Concept +from ..models.Template import Template from ..models.GenericEntity import GenericEntity from ..models.PublishedConcept import PublishedConcept from ..models.PublishedGenericEntity import PublishedGenericEntity @@ -34,7 +35,39 @@ def wrap(request, *args, **kwargs): return wrap -""" Status helpers """ + +""" Render helpers """ + +def should_render_template(template=None, **kwargs): + """ + Method to det. whether a template should be renderable + based on its `hide_on_create` property + + Args: + template {model}: optional parameter to check a model instance directly + + **kwargs (any): params to use when querying the template model + + Returns: + A boolean reflecting the renderable status of a template model + + """ + if template is None: + if len(kwargs.keys()) < 1: + return False + + template = Template.objects.filter(**kwargs) + + if template.exists(): + template = template.first() + + if not isinstance(template, Template) or not hasattr(template, 'hide_on_create'): + return False + + return not template.hide_on_create + + +""" Status helpers """ def is_member(user, group_name): """ @@ -231,11 +264,14 @@ def get_accessible_entities( ) entities = entities.filter(query) \ + .exclude( + template__hide_on_create=True + ) \ .annotate( was_deleted=Subquery( GenericEntity.objects.filter( id=OuterRef('id'), - is_deleted=True + is_deleted=True, ) \ .values('id') ) @@ -249,7 +285,8 @@ def get_accessible_entities( return entities.distinct('id') entities = entities.filter( - publish_status=APPROVAL_STATUS.APPROVED + publish_status=APPROVAL_STATUS.APPROVED, + template__hide_on_create=False, ) \ .annotate( was_deleted=Subquery( @@ -264,6 +301,126 @@ def get_accessible_entities( return entities.distinct('id') +def get_latest_owner_version_from_concept(phenotype_id, concept_id, concept_version_id=None, default=None): + """ + Gets the latest phenotype owner version id from a given concept + and its expected owner id + + Args: + phenotype_id (int): The phenotype owner id + + concept_id (int): The child concept id + + concept_version_id (int): An optional child concept version id + + default (any): An optional default return value + + Returns: + Returns either (a) an integer representing the version id + OR; (b) the optional default value + """ + latest_version = default + with connection.cursor() as cursor: + sql = None + params = { 'phenotype_id': phenotype_id, 'concept_id': concept_id } + if isinstance(concept_version_id, int): + sql = ''' + + with + phenotype_children as ( + select id as phenotype_id, + history_id as phenotype_version_id, + cast(concepts->>'concept_id' as integer) as concept_id, + cast(concepts->>'concept_version_id' as integer) as concept_version_id + from ( + select id, + history_id, + concepts + from public.clinicalcode_historicalgenericentity as entity, + json_array_elements(entity.template_data::json->'concept_information') as concepts + where json_array_length(entity.template_data::json->'concept_information') > 0 + and id = %(phenotype_id)s + ) hge_concepts + where (concepts->>'concept_id')::int = %(concept_id)s + ), + priorities as ( + select t1.*, 1 as sel_priority + from phenotype_children as t1 + where t1.concept_version_id = %(concept_version_id)s + union all + select t2.*, 2 as sel_priority + from phenotype_children as t2 + ), + sorted_ref as ( + select phenotype_id, + phenotype_version_id, + concept_id, + concept_version_id, + row_number() over ( + partition by concept_version_id + order by sel_priority + ) as reference + from priorities + ) + + select phenotype_id, + max(phenotype_version_id) as phenotype_version_id + from ( + select * + from sorted_ref + where reference = 1 + ) as pheno + join public.clinicalcode_historicalgenericentity as entity + on pheno.phenotype_id = entity.id + and pheno.phenotype_version_id = entity.history_id + group by phenotype_id; + + ''' + + params.update({ 'concept_version_id': concept_version_id }) + else: + sql = ''' + + with + phenotype_children as ( + select id as phenotype_id, + history_id as phenotype_version_id, + cast(concepts->>'concept_id' as integer) as concept_id, + cast(concepts->>'concept_version_id' as integer) as concept_version_id + from ( + select id, + history_id, + concepts + from public.clinicalcode_historicalgenericentity as entity, + json_array_elements(entity.template_data::json->'concept_information') as concepts + where json_array_length(entity.template_data::json->'concept_information') > 0 + and id = %(phenotype_id)s + ) hge_concepts + where (concepts->>'concept_id')::int = %(concept_id)s + ) + + select phenotype_id, + max(phenotype_version_id) as phenotype_version_id + from phenotype_children as pheno + join public.clinicalcode_historicalgenericentity as entity + on pheno.phenotype_id = entity.id + and pheno.phenotype_version_id = entity.history_id + group by phenotype_id; + + ''' + + cursor.execute(sql, params=params) + + columns = [col[0] for col in cursor.description] + row = cursor.fetchone() + + if row is not None: + row = dict(zip(columns, row)) + latest_version = row.get('phenotype_version_id') + + return latest_version + + def get_accessible_concepts( request, group_permissions=[GROUP_PERMISSIONS.VIEW, GROUP_PERMISSIONS.EDIT] diff --git a/CodeListLibrary_project/clinicalcode/entity_utils/template_utils.py b/CodeListLibrary_project/clinicalcode/entity_utils/template_utils.py index 54f3d1d2e..7e373be0c 100644 --- a/CodeListLibrary_project/clinicalcode/entity_utils/template_utils.py +++ b/CodeListLibrary_project/clinicalcode/entity_utils/template_utils.py @@ -1,10 +1,13 @@ from django.apps import apps from django.db.models import Q, ForeignKey +from django.db.models.query import QuerySet -from . import filter_utils from . import concept_utils +from . import filter_utils from . import constants +from ..models.OntologyTag import OntologyTag + def try_get_content(body, key, default=None): """ Attempts to get content within a dict by a key, if it fails to do so, returns the default value @@ -470,39 +473,50 @@ def get_template_sourced_values(template, field, default=None, request=None): 'name': v, 'value': i }) - + return output elif 'source' in validation: source_info = validation.get('source') - try: - model = apps.get_model(app_label='clinicalcode', model_name=source_info.get('table')) - - column = 'id' - if 'query' in source_info: - column = source_info['query'] - - if 'filter' in source_info: - filter_query = try_get_filter_query(field, source_info.get('filter'), request=request) - query = {**filter_query} - else: - query = { } - - queryset = model.objects.filter(Q(**query)) - if queryset.exists(): - relative = 'name' - if 'relative' in source_info: - relative = source_info['relative'] - - output = [] - for instance in queryset: - output.append({ - 'name': getattr(instance, relative), - 'value': getattr(instance, column) - }) - - return output if len(output) > 0 else default - except: - pass + if not source_info: + return default + + model_name = source_info.get('table') + tree_models = source_info.get('trees') + + if isinstance(tree_models, list): + output = OntologyTag.get_groups(tree_models, default=default) + if isinstance(output, list): + return output + elif isinstance(model_name, str): + try: + model = apps.get_model(app_label='clinicalcode', model_name=model_name) + + column = 'id' + if 'query' in source_info: + column = source_info.get('query') + + if 'filter' in source_info: + filter_query = try_get_filter_query(field, source_info.get('filter'), request=request) + query = {**filter_query} + else: + query = { } + + queryset = model.objects.filter(Q(**query)) + if queryset.exists(): + relative = 'name' + if 'relative' in source_info: + relative = source_info.get('relative') + + output = [] + for instance in queryset: + output.append({ + 'name': getattr(instance, relative), + 'value': getattr(instance, column) + }) + + return output if len(output) > 0 else default + except: + pass return default @@ -638,7 +652,16 @@ def get_template_data_values(entity, layout, field, hide_user_details=False, req if output is not None: return [output] elif field_type == 'int_array': - if 'source' in validation: + source_info = validation.get('source') + if not source_info: + return default + + model_name = source_info.get('table') + tree_models = source_info.get('trees') + + if isinstance(tree_models, list): + return OntologyTag.get_detailed_source_value(data, tree_models, default=default) + elif isinstance(model_name, str): values = [ ] for item in data: value = get_detailed_sourced_value(item, info) diff --git a/CodeListLibrary_project/clinicalcode/generators/__init__.py b/CodeListLibrary_project/clinicalcode/generators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/CodeListLibrary_project/clinicalcode/generators/graphs/__init__.py b/CodeListLibrary_project/clinicalcode/generators/graphs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/CodeListLibrary_project/clinicalcode/generators/graphs/constants.py b/CodeListLibrary_project/clinicalcode/generators/graphs/constants.py new file mode 100644 index 000000000..b1a0fc359 --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/generators/graphs/constants.py @@ -0,0 +1,5 @@ +import enum + +class GraphTypes(str, enum.Enum): + Tree = 0 + DirectedAcyclicGraph = 1 diff --git a/CodeListLibrary_project/clinicalcode/generators/graphs/generator.py b/CodeListLibrary_project/clinicalcode/generators/graphs/generator.py new file mode 100644 index 000000000..834cc1022 --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/generators/graphs/generator.py @@ -0,0 +1,85 @@ +import json + +from . import utils +from . import constants + +class Graph: + """ + [!] Note: see utils.py for kwargs to change size / connectivity + + e.g. + + ```py + # Generate a graph + graph = Graph.generate(graph_type=Graph.Types.DirectedAcyclicGraph) # or Graph.generate(graph_type=Graph.Types.Tree) + + # `graph.dots` - can be used to generate the edge list in DOT format, ref @ https://en.wikipedia.org/wiki/DOT_(graph_description_language + print(graph.dots) + + # `graph.nodes` - provides the fake data associated with each node and its edge list + print(graph.nodes) + + # `graph.dump` - dump to file if needed + graph.dump(output_file='./test.json') + ``` + + """ + + __key = object() + + @classmethod + def generate(cls, graph_type, **kwargs): + return Graph(cls.__key, graph_type, **kwargs) + + @classmethod + @property + def Types(cls): + return constants.GraphTypes + + def __init__(self, key, graph_type, **kwargs): + if key != Graph._Graph__key: + raise AssertionError('Constructor is private, please use the `generate` method') + + if graph_type == constants.GraphTypes.DirectedAcyclicGraph: + self.network = utils.generate_dag(**kwargs) + elif graph_type == constants.GraphTypes.Tree: + self.network = utils.generate_tree(**kwargs) + else: + raise NotImplementedError('Graph type is not implemented') + self.type = graph_type + + @property + def dots(self): + dots = '' + for edge in self.network: + dots += '\t%(index)s -> %(vertex)s;\n' % { 'index': edge[0], 'vertex': edge[1] } + + return 'digraph {\n%s}' % dots + + @property + def nodes(self): + from faker import Faker + + fake = Faker() + nodes = [ ] + for edge in self.network: + node = next((x for x in nodes if x['id'] == edge[0]), None) + if not node: + node = { 'id': edge[0], 'name': fake.name(), 'edges': [ ] } + nodes.append(node) + + connection = next((x for x in nodes if x['id'] == edge[1]), None) + if not connection: + nodes.append({ 'id': edge[1], 'name': fake.name(), 'edges': [ ] }) + + node['edges'].append(edge[1]) + + return nodes + + def dump(self, output_file=None, indent=2): + nodes = self.nodes + if isinstance(output_file, str): + with open(output_file, 'w') as f: + json.dump(nodes, f, indent=indent) + + return json.dumps(nodes, indent=indent) diff --git a/CodeListLibrary_project/clinicalcode/generators/graphs/utils.py b/CodeListLibrary_project/clinicalcode/generators/graphs/utils.py new file mode 100644 index 000000000..10ff75f26 --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/generators/graphs/utils.py @@ -0,0 +1,42 @@ +import random + +def generate_dag(connectivity=0.5, min_rank_width=2, max_rank_width=4, min_rank_height=3, max_rank_height=5): + ranks = random.randint(min_rank_height, max_rank_height) + nodes = 0 + node_counter = 0 + network = [] + rank_list = [] + + for i in range(ranks): + new_nodes = random.randint(min_rank_width, max_rank_width) + + ranks = [] + for j in range(new_nodes): + ranks.append(node_counter) + node_counter += 1 + rank_list.append(ranks) + + if i > 0: + for j in rank_list[i - 1]: + for k in range(new_nodes): + if random.random() < connectivity: + network.append((j, k+nodes)) + + nodes += new_nodes + + return network + +def generate_tree(size=10): + sequence = [random.choice(range(size)) for i in range(size - 2)] + height = len(sequence) + L = set(range(height+2)) + network = [] + + for i in range(height): + u, v = sequence[0], min(L - set(sequence)) + sequence.pop(0) + L.remove(v) + network.append((u,v)) + network.append((L.pop(), L.pop())) + + return network diff --git a/CodeListLibrary_project/clinicalcode/management/commands/dag_tasks.py b/CodeListLibrary_project/clinicalcode/management/commands/dag_tasks.py new file mode 100644 index 000000000..4d27d2019 --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/management/commands/dag_tasks.py @@ -0,0 +1,553 @@ +from django.core.management.base import BaseCommand +from django.db import transaction, connection + +import re +import os +import json +import enum +import time + +from ...entity_utils import constants +from ...models.CodingSystem import CodingSystem +from ...models.OntologyTag import OntologyTagEdge, OntologyTag +from ...generators.graphs.generator import Graph as GraphGenerator + + +###################################################### +# # +# Constants # +# # +###################################################### +class IterableMeta(enum.EnumMeta): + """ + Metaclass that defines additional methods + of operation and interaction with enums + + """ + def from_name(cls, name): + if name in cls: + return getattr(cls, name) + + def __contains__(cls, lhs): + try: + cls(lhs) + except ValueError: + return lhs in cls.__members__.keys() + else: + return True + +class GraphType(int, enum.Enum, metaclass=IterableMeta): + """ + Parsed from input file to determine how to handle the data + + e.g. { type: 'CODE_CATEGORIES' } within `./data/graphs/icd10_categories.json` + + """ + CODE_CATEGORIES = 0 + ANATOMICAL_CATEGORIES = 1 + SPECIALITY_CATEGORIES = 2 + +class LogType(int, enum.Enum, metaclass=IterableMeta): + """ + Enum that reflects the output style, as described by the BaseCommand log style + + See ref @ https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/#django.core.management.BaseCommand.style + + """ + SUCCESS = 1 + NOTICE = 2 + WARNING = 3 + ERROR = 4 + + +###################################################### +# # +# Graph Builders # +# # +###################################################### +class GraphBuilders: + """ + Builds a graph according to the given GraphType + and its associated data + + """ + + @classmethod + def try_build(cls, builder_type, data): + """ + Attempts to build a graph given a valid builder type + + """ + if not isinstance(builder_type, GraphType): + return False, 'Expected valid GraphType, got %s' % str(builder_type) + + desired_builder = getattr(cls, builder_type.name) + if desired_builder is None: + return False, 'Invalid Builder, no class method available with the name: %s' % builder_type.name + + bound_to = getattr(desired_builder, '__self__', None) + if not isinstance(bound_to, type) or bound_to is not cls: + return False, 'Invalid Builder, no appropriate class method found for BuilderType<%s>' % builder_type.name + + return desired_builder(data) + + @classmethod + def CODE_CATEGORIES(cls, data): + """ + ICD-10 Disease Category builder test(s) + + Note: + + ICD-10 codes were scraped from the ICD-10 classification website, + and matched with the Atlas phecodes + + This builder generates a DAG of ICD-10 codes, matched with the codes + within our database and selects the appropriate CodingSystem + + """ + + ''' [!] Warning: This is only partially optimised ''' + if not isinstance(data, list): + return False, 'Invalid data type, expected list but got %s' % type(data) + + # const + icd_10_id = CodingSystem.objects.get(name='ICD10 codes').id + + # process nodes + nodes = [ ] + result = [ ] + linkage = [ ] + name_hashmap = { } + started = time.time() + + def create_linkage(parent, parent_index, children): + for child_data in children: + name = child_data.get('name').strip() + name = re.sub(r'\((\b(?=[a-zA-Z\d]+)[a-zA-Z]*\d[a-zA-Z\d]*-\b(?=[A-Z\d]+)[a-zA-Z]*\d[a-zA-Z\d]*)\)', '', name).strip() + code = child_data.get('code').strip() + + # ICD-10 uses non-unique names, add code to vary them if required + if name in name_hashmap: + name = f'{name} ({code})' + name_hashmap[name] = True + + # Create child node and process descendants + properties = { 'code': code, 'coding_system_id': icd_10_id } + + node = OntologyTag(name=name, type_id=constants.ONTOLOGY_TYPES.CLINICAL_DISEASE, properties=properties) + index = len(nodes) + nodes.append(node) + linkage.append([parent_index, index]) + + descendants = child_data.get('children') + child_count = len(descendants) if isinstance(descendants, list) else 0 + result.append(f'\t\tChildDiseaseNode') + + if isinstance(descendants, list): + create_linkage(node, index, descendants) + + for root_data in data: + # clean up the section name(s) from scraped data + root_name = root_data.get('name').strip() + matched_code = re.search(r'(\b(?=[a-zA-Z\d]+)[a-zA-Z]*\d[a-zA-Z\d]*-\b(?=[A-Z\d]+)[a-zA-Z]*\d[a-zA-Z\d]*)', root_name) + + root_name = re.sub(r'\((\b(?=[a-zA-Z\d]+)[a-zA-Z]*\d[a-zA-Z\d]*-\b(?=[A-Z\d]+)[a-zA-Z]*\d[a-zA-Z\d]*)\)', '', root_name).strip() + derived_code = matched_code.group() if matched_code else None + + # process node and its branches + properties = { 'code': derived_code, 'coding_system_id': icd_10_id } + + root = OntologyTag(name=root_name, type_id=constants.ONTOLOGY_TYPES.CLINICAL_DISEASE, properties=properties) + index = len(nodes) + nodes.append(root) + + children = root_data.get('sections') + result.append(f'\tRootDiseaseNode') + + create_linkage(root, index, children) + + with transaction.atomic(): + # bulk create nodes & children + nodes = OntologyTag.objects.bulk_create(nodes) + + # bulk create edges + OntologyTag.children.through.objects.bulk_create( + [ + # list comprehension here is required because we need to match the instance(s) + OntologyTag.children.through( + name=f'{nodes[link[0]].name} | {nodes[link[1]].name}', + parent=nodes[link[0]], + child=nodes[link[1]] + ) + for link in linkage + ], + batch_size=7000 + ) + + # update coding system and apply related code + with connection.cursor() as cursor: + ''' [!] Note: We could probably optimise this? ''' + + sql = """ + -- update matched values + update public.clinicalcode_ontologytag as trg + set properties = properties || jsonb_build_object('code_id', src.code_id) + from ( + select node.id as node_id, + code.id as code_id + from public.clinicalcode_ontologytag as node + join public.clinicalcode_icd10_codes_and_titles_and_metadata as code + on replace(node.properties->>'code'::text, '.', '') = replace(code.code, '.', '') + where node.properties is not null + and node.properties ? 'code' + and node.type_id = %(type_id)s + ) src + where trg.id = src.node_id + and trg.type_id = %(type_id)s + and trg.properties is not null; + """ + cursor.execute(sql, { 'type_id': constants.ONTOLOGY_TYPES.CLINICAL_DISEASE.value }) + + # create result string for log + elapsed = (time.time() - started) + result = 'Created DiseaseNodes {\n%s\n}' % (icd_10_id, elapsed, '\n'.join(result)) + + return True, result + + @classmethod + def ANATOMICAL_CATEGORIES(cls, data): + """ + Anatomical category builder + + Note: + + Currently, there are no known links between anatomical categories + provided by the Atlas dataset. + + As such, this method creates a tree without any children. + + """ + + if not isinstance(data, list): + return False, 'Invalid data type, expected list but got %s' % type(data) + + # process nodes + nodes = [ ] + result = [ ] + started = time.time() + + for root_node in data: + node_id = root_node.get('id') + node_name = root_node.get('name') + + if not isinstance(node_id, int) or not isinstance(node_name, str): + err = 'Failed to create Node, expected but got Node' \ + % (type(node_id), type(node_name)) + return False, err + + node = OntologyTag(name=node_name.strip(), atlas_id=node_id, type_id=constants.ONTOLOGY_TYPES.CLINICAL_FUNCTIONAL_ANATOMY) + nodes.append(node) + result.append(f'\tAnatomicalRootNode') + + # bulk create nodes + with transaction.atomic(): + nodes = OntologyTag.objects.bulk_create(nodes) + + # create result string for log + elapsed = (time.time() - started) + result = 'Created AnatomicalNodes {\n%s\n}' % (elapsed, len(nodes), '\n'.join(result)) + + return True, result + + @classmethod + def SPECIALITY_CATEGORIES(cls, data): + """ + Clinical domain builder + + Note: + + This speciality data was scraped from the Atlas datasources, + it creates a tree hierarchy of clinical specialities and subspecialities + + DAG required as there are some specialities with overlap, e.g.: + + - Pre-hospital Emergency Medicine as as child of Anaesthetics, ICM and EM + - Paediatric Intensive Care Medicine as a child of Paediatrics and ICM + + """ + + if not isinstance(data, dict): + return False, 'Invalid data type, expected list but got %s' % type(data) + + # process nodes + nodes = [ ] + result = [ ] + linkage = [ ] + started = time.time() + + for root_key, children in data.items(): + root_name = root_key.strip() + root_node = OntologyTag(name=root_name, type_id=constants.ONTOLOGY_TYPES.CLINICAL_DOMAIN) + + root_index = len(nodes) + nodes.append(root_node) + result.append(f'\tSpecialityRootNode') + + if len(children) > 0: + for child_key in children: + child_name = child_key.strip() + + related_index = next((i for i, e in enumerate(nodes) if e.name == child_name), None) + if related_index is None: + related_index = len(nodes) + child = OntologyTag(name=child_name, type_id=constants.ONTOLOGY_TYPES.CLINICAL_DOMAIN) + nodes.append(child) + + linkage.append([root_index, related_index]) + result.append(f'\t\tChildSpecialityNode') + + # bulk create nodes & children + with transaction.atomic(): + nodes = OntologyTag.objects.bulk_create(nodes) + + # bulk create edges + OntologyTag.children.through.objects.bulk_create( + [ + # list comprehension here is required because we need to match the instance(s) + OntologyTag.children.through( + name=f'{nodes[link[0]].name} | {nodes[link[1]].name}', + parent=nodes[link[0]], + child=nodes[link[1]] + ) + for link in linkage + ], + batch_size=7000 + ) + + # create result string for log + elapsed = (time.time() - started) + result = 'Created SpecialityNodes {\n%s\n}' % (elapsed, '\n'.join(result)) + + return True, result + + +###################################################### +# # +# DAG Command # +# # +###################################################### +class Command(BaseCommand): + help = 'Various tasks associated with the generation of DAGs' + + DEFAULT_FILE = 'data/graphs/categories.json' + VALID_FILE_TYPES = ['.json'] + LOG_FILE_NAME = 'DAG_LOGS' + LOG_FILE_EXT = '.txt' + + def __get_log_style(self, style): + """ + Returns the BaseCommand's log style + + See ref @ https://docs.djangoproject.com/en/5.0/howto/custom-management-commands/#django.core.management.BaseCommand.style + + """ + if isinstance(style, str): + style = style.upper() + if style in LogType.__members__: + return getattr(self.style, style) + elif isinstance(style, LogType): + if style.name in LogType.__members__: + return getattr(self.style, style.name) + return self.style.SUCCESS + + def __log_dots(self, nodes, name=None): + """ + Logs the edge list in DOT format + + See ref @ https://en.wikipedia.org/wiki/DOT_(graph_description_language + + """ + name = name or 'Unknown' + dots = '' + for i, node in enumerate(nodes): + for child in node.children.all(): + dots += '\t%(index)s -> %(vertex)s;\n' % { 'index': node.id, 'vertex': child.id } + + self.__log_to_file('Digraph<%s>: digraph {\n%s}' % (name, dots)) + + def __log_to_file(self, message, style=LogType.SUCCESS): + """ + Logs the message, prepended with its style, to the log file (if a valid directory has been provided) + + """ + directory = self._log_dir + if not isinstance(directory, str): + return + + if not os.path.isabs(directory): + directory = os.path.join( + os.path.abspath(os.path.dirname('manage.py')), + directory + ) + + if not os.path.exists(directory): + os.makedirs(directory) + + style = style.name if isinstance(style, LogType) else style + filename = os.path.join(directory, f'{self.LOG_FILE_NAME}{self.LOG_FILE_EXT}') + offset = '\n' if os.path.exists(filename) else '' + with open(filename, 'a') as file: + file.writelines([f'{offset}[{style}] {message}\n']) + + def __log(self, message, style=LogType.SUCCESS): + """ + Logs the incoming to: + 1. The log file, if a valid directory has been provided + 2. The terminal, if the `-p` argument has been provided + + """ + self.__log_to_file(message, style) + + if not self._verbose: + return + style = self.__get_log_style(style) + self.stdout.write(style(message)) + + def __try_load_file(self, filepath): + """ + Attempts to load the given filepath as a JSON object + + """ + filepath = os.path.join( + os.path.abspath(os.path.dirname('manage.py')), + filepath if filepath is not None else self.DEFAULT_FILE + ) + + self.__log(f'Initialising DAG command with Path<{filepath}> ...') + + if not os.path.exists(filepath): + self.__log(f'Path<{filepath}> does not exist', LogType.ERROR) + return + + if not os.path.isfile(filepath): + self.__log(f'Path<{filepath}> does not reference a file', LogType.ERROR) + return + + file_extension = os.path.splitext(filepath)[1] + if file_extension not in self.VALID_FILE_TYPES: + self.__log(f'File<{filepath}> does not reference a valid file of expected types: {", ".join(self.VALID_FILE_TYPES)}', LogType.ERROR) + return + + try: + with open(filepath) as f: + data = json.load(f) + return data + except Exception as e: + self.__log(f'Error when attempting to load File<{filepath}>:\n{str(e)}', LogType.ERROR) + return None + + def __try_build_dag(self, filepath): + """ + Attempts to build the DAG from the given filepath + + """ + # attempt import + data = self.__try_load_file(filepath) + if data is None: + return + + # validate + graph_input = data.get('data', None) + if graph_input is None: + self.__log(f'No property `data` found within File<{filepath}>', LogType.ERROR) + return + + builder_type = data.get('type', None) + builder_type = GraphType[builder_type] if builder_type is not None and builder_type in GraphType else None + + if not isinstance(builder_type, GraphType): + self.__log(f'No valid property `type` found within File<{filepath}>', LogType.ERROR) + return + + # attempt generation + success, result = GraphBuilders.try_build(builder_type, graph_input) + if not success: + result = result if isinstance(result, str) else 'Unknown error occurred' + self.__log(f'Error occurred when processing File<{filepath}> via BuilderType<{builder_type.name}>:\n\t{result}', LogType.ERROR) + return + + self.__log('Building Graph from File<%s> was completed successfully' % filepath, LogType.SUCCESS) + + if isinstance(result, str): + self.__log_to_file(result, LogType.SUCCESS) + + def __generate_debug_dag(self): + """ + Responsible for generating a debug dag using the graph generators & its utility methods + + """ + graph = GraphGenerator.generate(graph_type=GraphGenerator.Types.DirectedAcyclicGraph) + nodes = [ + OntologyTag( + name=node.get('name'), + type_id=constants.ONTOLOGY_TYPES.CLINICAL_DISEASE, + properties={'code': str(node.get('id')), 'coding_system_id': 4} + ) + for node in graph.nodes + ] + nodes = OntologyTag.objects.bulk_create(nodes) + + output = '' + for i, data in enumerate(graph.nodes): + index = str(data.get('id')) + edges = data.get('edges') + + node = next((x for x in nodes if x.code == index), None) + if not node: + continue + + output = f'{output}\n\tNode [' + if len(edges) > 0: + for j, element in enumerate(edges): + connection = next((x for x in nodes if x.properties.get('code') == str(element)), None) + if not connection: + continue + output = f'{output}\n\t\tConnection' + node.add_child(connection) + output = output + '\n\t]' + else: + output = output + ' ]' + self.__log('Graph Generation {%s\n}' % output) + + if self._log_dir: + self.__log_dots(nodes=nodes, name='DebugGraph') + + def add_arguments(self, parser): + """ + Handles arguments given via the CLI + + """ + parser.add_argument('-p', '--print', type=bool, help='Print debug information to the terminal') + parser.add_argument('-f', '--file', type=str, help='Location of DAG data relative to manage.py') + parser.add_argument('-d', '--debug', type=bool, help='If true, attempts to generate DAG and ignores the --file parameter') + parser.add_argument('-l', '--log', type=str, help=f'Expects directory, will output logs incl. DOTS representation to file as {self.LOG_FILE_NAME}{self.LOG_FILE_EXT}') + + def handle(self, *args, **kwargs): + """ + Main command handle + + """ + # init parameters + verbose = kwargs.get('print') + filepath = kwargs.get('file') + is_debug = kwargs.get('debug') + log_file = kwargs.get('log') + + # det. handle + self._verbose = verbose + self._log_dir = log_file if isinstance(log_file, str) and len(log_file) > 0 else None + + if is_debug: + self.__generate_debug_dag() + else: + self.__try_build_dag(filepath or self.DEFAULT_FILE) diff --git a/CodeListLibrary_project/clinicalcode/migrations/0113_ontologytag_ontologytagedge_ontologytag_children.py b/CodeListLibrary_project/clinicalcode/migrations/0113_ontologytag_ontologytagedge_ontologytag_children.py new file mode 100644 index 000000000..0d59a9b2a --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/migrations/0113_ontologytag_ontologytagedge_ontologytag_children.py @@ -0,0 +1,44 @@ +# Generated by Django 4.1.10 on 2024-03-04 11:35 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('clinicalcode', '0112_dmd_codes'), + ] + + operations = [ + migrations.CreateModel( + name='OntologyTag', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=1024)), + ('type_id', models.IntegerField(choices=[('CLINICAL_DISEASE', 0), ('CLINICAL_DOMAIN', 1), ('CLINICAL_FUNCTIONAL_ANATOMY', 2)])), + ('atlas_id', models.IntegerField(blank=True, null=True)), + ('properties', models.JSONField(blank=True, null=True)), + ], + options={ + 'ordering': ('type_id', 'id'), + }, + ), + migrations.CreateModel( + name='OntologyTagEdge', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=2048)), + ('child', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='parent_edges', to='clinicalcode.ontologytag')), + ('parent', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='children_edges', to='clinicalcode.ontologytag')), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddField( + model_name='ontologytag', + name='children', + field=models.ManyToManyField(blank=True, related_name='parents', through='clinicalcode.OntologyTagEdge', to='clinicalcode.ontologytag'), + ), + ] diff --git a/CodeListLibrary_project/clinicalcode/migrations/0114_ontologytag_clinicalcod_id_d32433_idx_and_more.py b/CodeListLibrary_project/clinicalcode/migrations/0114_ontologytag_clinicalcod_id_d32433_idx_and_more.py new file mode 100644 index 000000000..6c7e26888 --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/migrations/0114_ontologytag_clinicalcod_id_d32433_idx_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 4.1.10 on 2024-03-05 09:06 + +import django.contrib.postgres.indexes +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('clinicalcode', '0113_ontologytag_ontologytagedge_ontologytag_children'), + ] + + operations = [ + migrations.AddIndex( + model_name='ontologytag', + index=models.Index(fields=['id', 'type_id'], name='clinicalcod_id_d32433_idx'), + ), + migrations.AddIndex( + model_name='ontologytag', + index=django.contrib.postgres.indexes.GinIndex(fields=['name'], name='ot_name_gin_idx'), + ), + ] diff --git a/CodeListLibrary_project/clinicalcode/models/GenericEntity.py b/CodeListLibrary_project/clinicalcode/models/GenericEntity.py index 81e072242..99d26674c 100644 --- a/CodeListLibrary_project/clinicalcode/models/GenericEntity.py +++ b/CodeListLibrary_project/clinicalcode/models/GenericEntity.py @@ -78,6 +78,7 @@ class GenericEntity(models.Model): ''' Historical data ''' history = HistoricalRecords() + @transaction.atomic def save(self, ignore_increment=False, *args, **kwargs): """ [!] Note: @@ -90,25 +91,23 @@ def save(self, ignore_increment=False, *args, **kwargs): entity_class = getattr(template_layout, 'entity_class') if entity_class is not None: if ignore_increment: - with transaction.atomic(): - entitycls = EntityClass.objects.select_for_update().get(pk=entity_class.id) - entity_id = gen_utils.parse_int( - self.id.replace(entitycls.entity_prefix, ''), - default=None - ) - - if not entity_id: - raise ValidationError('Unable to parse entity id') - - if entitycls.entity_count < entity_id: - entitycls.entity_count = entity_id - entitycls.save() - elif not self.pk and not ignore_increment: - with transaction.atomic(): - entitycls = EntityClass.objects.select_for_update().get(pk=entity_class.id) - index = entitycls.entity_count = entitycls.entity_count + 1 - self.id = f'{entitycls.entity_prefix}{index}' + entitycls = EntityClass.objects.select_for_update().get(pk=entity_class.id) + entity_id = gen_utils.parse_int( + self.id.replace(entitycls.entity_prefix, ''), + default=None + ) + + if not entity_id: + raise ValidationError('Unable to parse entity id') + + if entitycls.entity_count < entity_id: + entitycls.entity_count = entity_id entitycls.save() + elif not self.pk and not ignore_increment: + entitycls = EntityClass.objects.select_for_update().get(pk=entity_class.id) + index = entitycls.entity_count = entitycls.entity_count + 1 + self.id = f'{entitycls.entity_prefix}{index}' + entitycls.save() if self.template_data and 'version' in self.template_data: self.template_version = self.template_data.get('version') diff --git a/CodeListLibrary_project/clinicalcode/models/OntologyTag.py b/CodeListLibrary_project/clinicalcode/models/OntologyTag.py new file mode 100644 index 000000000..4350a7af9 --- /dev/null +++ b/CodeListLibrary_project/clinicalcode/models/OntologyTag.py @@ -0,0 +1,651 @@ +from django.apps import apps +from django.db import models, transaction, connection +from django.db.models import F, Count, Max, Case, When, Exists, OuterRef +from django.db.models.query import QuerySet +from django.db.models.functions import JSONObject +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.aggregates.general import ArrayAgg +from django_postgresql_dag.models import node_factory, edge_factory + +from ..entity_utils import gen_utils +from ..entity_utils import constants + +from .CodingSystem import CodingSystem + +class OntologyTagEdge(edge_factory('OntologyTag', concrete=False)): + """ + OntologyTagEdge + + This class describes the parent-child relationship + between two nodes - as defined by its `parent_id` + and its `child_id` + + """ + + # Fields + name = models.CharField(max_length=2048, unique=False) + + # Dunder methods + def __str__(self): + return self.name + + # Public methods + def save(self, *args, **kwargs): + """ + Save override to appropriately style the instance's + name field + """ + self.name = f'{self.parent.name} {self.child.name}' + super().save(*args, **kwargs) + +class OntologyTag(node_factory(OntologyTagEdge)): + """ + OntologyTag + + This class describes the fields for each node + within an ontology group, separated by its `type_id` + + The relationship between nodes are described by + the associated `OntologyTagEdge` instances, describing + each node's relationship with another + + Together, these classes allow you to describe a graph + of nodes that could be described as either a Directed + Acyclic Graph or a Hierarchical Tree + + Ref: + - Hierarchical Tree @ https://en.wikipedia.org/wiki/Tree_structure + - Directed Acyclic Graph @ https://en.wikipedia.org/wiki/Directed_acyclic_graph + + """ + + # Fields + name = models.CharField(max_length=1024, unique=False) + type_id = models.IntegerField(choices=[(e.name, e.value) for e in constants.ONTOLOGY_TYPES]) + atlas_id = models.IntegerField(blank=True, null=True, unique=False) + properties = models.JSONField(blank=True, null=True) + + # Metadata + class Meta: + ordering = ('type_id', 'id', ) + + indexes = [ + models.Index(fields=['id', 'type_id']), + GinIndex( + name='ot_name_gin_idx', + fields=['name'] + ), + ] + + # Dunder methods + def __str__(self): + return self.name + + # Private methods + def __validate_disease_code_id(self, properties, default=None): + """ + Attempts to validate the associated code id, should only + be called for ontology instances that have a type_id + which associates an instance with a specific code, + e.g. ICD-10 category codes + + Args: + self (>): this instance + + properties (dict): the properties field associated + with this instance + + default (any|None): the default return value + + Returns: + Either (a) the default value if none are found, + or (b) the pk of the associated code + """ + if not isinstance(properties, dict): + return default + + code = None + try: + desired_code = properties.get('code') + desired_system_id = gen_utils.parse_int(properties.get('coding_system_id'), None) + + if not isinstance(desired_code, str) or not isinstance(desired_system_id, int): + return default + + desired_system = CodingSystem.objects.filter(pk__eq=desired_system_id) + desired_system = desired_system.first() if desired_system.exists() else None + + if desired_system is None: + return default + + comparators = [ desired_code.lower(), desired_code.replace('.', '').lower() ] + table_name = desired_system.table_name + model_name = desired_system.table_name.replace('clinicalcode_', '') + codes_name = desired_system.code_column_name.lower() + + query = """ + select * + from public.%(table_name)s + where lower(%(column_name)s); + + """ % { 'table_name': table_name, 'column_name': codes_name } + + codes = apps.get_model(app_label='clinicalcode', model_name=model_name) + code = codes.objects.raw(query + ' = ANY(%(values)s::text[])', { 'values': comparators }) + except: + code = None + finally: + if code is None or not code.exists(): + return default + return code.first().pk + + # Public methods + @classmethod + def get_groups(cls, ontology_ids=None, default=None): + """ + Derives the tree model data given a list containing + the model source and its associated label + + If no ontology_id list is provided we will return + all root(s) of each type + + Args: + ontology_ids (int[]|enum[]|none): an optional list of ontology model ids + + default (any|None): the default return value + + Returns: + Either (a) the default value if none are found, + or (b) a list of dicts containing the associated + tree model data + + """ + if not isinstance(ontology_ids, list): + ontology_ids = OntologyTag.objects.all().distinct('type_id').values_list('type_id') + ontology_ids = list(ontology_ids) + + output = None + for ontology_id in ontology_ids: + model_source = None + if isinstance(ontology_id, constants.ONTOLOGY_TYPES): + model_source = ontology_id.value + elif isinstance(ontology_id, int) and ontology_id in constants.ONTOLOGY_TYPES: + model_source = ontology_id + + if not isinstance(model_source, int): + continue + + model_label = constants.ONTOLOGY_LABELS[constants.ONTOLOGY_TYPES(ontology_id)] + data = OntologyTag.get_group_data(model_source, model_label, default=default) + if not isinstance(data, dict): + continue + + if output is None: + output = [] + + output.append(data) + + return output + + @classmethod + def get_group_data(cls, model_source, model_label=None, default=None): + """ + Derives the tree model data given the model source name + + Args: + model_source (int|enum): the ontology id + + model_label (str|None): the associated model label + + default (any|None): the default return value + + Returns: + Either (a) the default value if none are found, + or (b) a dict containing the associated tree model data + """ + if isinstance(model_source, constants.ONTOLOGY_TYPES): + model_source = model_source.value + elif not isinstance(model_source, int) or model_source not in constants.ONTOLOGY_TYPES: + return default + + try: + model_roots = OntologyTag.objects.roots().filter(type_id=model_source) + model_roots_len = model_roots.count() if isinstance(model_roots, QuerySet) else 0 + if model_roots_len < 1: + return default + + model_roots = model_roots.values('id', 'name') \ + .annotate( + child_count=Count(F('children')), + max_parent_id=Max(F('parents')) + ) \ + .annotate( + tree_dataset=JSONObject( + id=F('id'), + label=F('name'), + properties=F('properties'), + isLeaf=Case( + When(child_count__lt=1, then=True), + default=False + ), + isRoot=Case( + When(max_parent_id__isnull=True, then=True), + default=False + ), + type_id=F('type_id'), + atlas_id=F('atlas_id'), + child_count=F('child_count') + ) + ) \ + .values_list('tree_dataset', flat=True) + except: + pass + else: + model_label = model_label or constants.ONTOLOGY_LABELS[constants.ONTOLOGY_TYPES(model_source)] + return { + 'model': { 'source': model_source, 'label': model_label }, + 'nodes': list(model_roots), + } + + return default + + @classmethod + def get_node_data(cls, node_id, ontology_id=None, model_label=None, default=None): + """ + Derives the ontology node data given the node id + + Args: + node_id (int): the node id + + ontology_id (int|enum|none): optional ontology model id + + default (any|None): the default return value + + Returns: + Either (a) the default value if none are found, + or (b) a dict containing the associated node's data + """ + model_source = None + if isinstance(ontology_id, constants.ONTOLOGY_TYPES): + model_source = ontology_id.value + elif isinstance(ontology_id, int) and ontology_id in constants.ONTOLOGY_TYPES: + model_source = ontology_id + + if not isinstance(node_id, int): + return default + + try: + node = None + if isinstance(model_source, int): + node = OntologyTag.objects.filter(id=node_id, type_id=model_source) + else: + node = OntologyTag.objects.filter(id=node_id) + + node = node.first() if node.exists() else None + if node is None: + return default + + model_source = node.type_id + parents = node.parents.all() + if parents.count() > 0: + parents = parents.annotate( + tree_dataset=JSONObject( + id=F('id'), + label=F('name'), + properties=F('properties'), + isRoot=Case( + When( + Exists(OntologyTag.parents.through.objects.filter( + child_id=OuterRef('pk'), + )), + then=False + ), + default=True + ), + isLeaf=False, + type_id=F('type_id'), + atlas_id=F('atlas_id'), + child_count=Count(F('children')), + parents=ArrayAgg('parents', distinct=True) + ) + ) \ + .values_list('tree_dataset', flat=True) + else: + parents = [] + + children = node.children.all() + if children.count() > 0: + children = OntologyTag.objects.filter(id__in=children) \ + .annotate( + child_count=Count(F('children')) + ) \ + .annotate( + tree_dataset=JSONObject( + id=F('id'), + label=F('name'), + properties=F('properties'), + isRoot=False, + isLeaf=Case( + When(child_count__lt=1, then=True), + default=False + ), + type_id=F('type_id'), + atlas_id=F('atlas_id'), + child_count=F('child_count'), + parents=ArrayAgg('parents', distinct=True) + ) + ) \ + .values_list('tree_dataset', flat=True) + else: + children = [] + + is_root = node.is_root() or node.is_island() + is_leaf = node.is_leaf() + + model_label = model_label or constants.ONTOLOGY_LABELS[constants.ONTOLOGY_TYPES(model_source)] + + result = { + 'id': node_id, + 'label': node.name, + 'model': { 'source': model_source, 'label': model_label }, + 'properties': node.properties, + 'isRoot': is_root, + 'isLeaf': is_leaf, + 'type_id': node.type_id, + 'atlas_id': node.atlas_id, + 'child_count': len(children), + 'parents': list(parents) if not isinstance(parents, list) else parents, + 'children': list(children) if not isinstance(children, list) else children, + } + + if not is_root: + roots = [ { 'id': x.id, 'label': x.name } for x in node.roots() ] + result.update({ 'roots': roots }) + + return result + except: + pass + + return default + + @classmethod + def build_tree(cls, descendant_ids, default=None): + """ + Attempts to derive the ontology tree data associated given a list of node + descendant ids + + Note: This is used to fill in the tree before sending it to the create page, + which allows for the selection(s) to be correctly displayed without + querying the entire tree + + Args: + descendant_ids (int[]): a list of node descendant ids + + default (any|None): the default return value + + Returns: + Either (a) the default value if we're unable to resolve the data, + or (b) a list of dicts containing the associated node's data + and its ancestry data + + """ + if not isinstance(descendant_ids, list): + return default + + ancestry = default + try: + with connection.cursor() as cursor: + sql = ''' + with + recursive ancestry(parent_id, child_id, depth, path) as ( + select n0.parent_id, + n0.child_id, + 1 as depth, + array[n0.parent_id, n0.child_id] as path + from public.clinicalcode_ontologytagedge as n0 + left outer join public.clinicalcode_ontologytagedge as n1 + on n0.parent_id = n1.child_id + where n0.child_id = any(%(node_ids)s) + union + select n2.parent_id, + ancestry.child_id, + ancestry.depth + 1 as depth, + n2.parent_id || ancestry.path + from ancestry + join public.clinicalcode_ontologytagedge as n2 + on n2.child_id = ancestry.parent_id + ), + ancestors as ( + select p0.child_id, + p0.path + from ancestry as p0 + join ( + select child_id, + max(depth) as max_depth + from ancestry + group by child_id + ) as lim + on lim.child_id = p0.child_id + and lim.max_depth = p0.depth + ), + objects as ( + select selected.child_id, + jsonb_build_object( + 'id', nodes.id, + 'label', nodes.name, + 'properties', nodes.properties, + 'isLeaf', case when count(edges1.child_id) < 1 then True else False end, + 'isRoot', case when max(edges0.parent_id) is NULL then True else False end, + 'type_id', nodes.type_id, + 'atlas_id', nodes.atlas_id, + 'child_count', count(edges1.child_id) + ) as tree + from ( + select id, + child_id + from ancestors, + unnest(path) as id + group by id, child_id + ) as selected + join public.clinicalcode_ontologytag as nodes + on nodes.id = selected.id + left outer join public.clinicalcode_ontologytagedge as edges0 + on nodes.id = edges0.child_id + left outer join public.clinicalcode_ontologytagedge as edges1 + on nodes.id = edges1.parent_id + group by selected.child_id, nodes.id + ) + + select ancestor.child_id, + ancestor.path, + json_agg(obj.tree) as dataset + from ancestors as ancestor + join objects as obj + on obj.child_id = ancestor.child_id + group by ancestor.child_id, ancestor.path; + ''' + + cursor.execute( + sql, + params={ 'node_ids': descendant_ids } + ) + + columns = [col[0] for col in cursor.description] + ancestry = [dict(zip(columns, row)) for row in cursor.fetchall()] + except: + pass + + return ancestry + + @classmethod + def get_full_names(cls, node, default=None): + """ + Derives the full name(s) of the root path + from a given node + + Args: + node (OntologyTag): the node instance + default (any): the default return value + + Returns: + Returns either (a) the full name string, if applicable; + OR (b) the default return value if no roots available + + """ + roots = [node.name for node in node.roots()] + if len(roots) > 0: + roots = '; '.join(roots) + roots = f'from {roots}' + return roots + + return default + + @classmethod + def get_detail_data(cls, node_ids, default=None): + """ + Attempts to derive the ontology data associated with a given a list of nodes + as described by a GenericEntity's template data + + [!] Note: No validation on the type of input is performed here, you are expected + to perform this validation prior to calling this method + + Args: + node_ids (int[]): a list of node ids + + default (any|None): the default return value + + Returns: + Either (a) the default value if we're unable to resolve the data, + OR; (b) a list of dicts containing the associated data + + """ + nodes = OntologyTag.objects.filter(id__in=node_ids) + roots = { node.id: OntologyTag.get_full_names(node) for node in nodes if not node.is_root() and not node.is_island() } + + nodes = nodes \ + .annotate( + child_count=Count(F('children')) + ) \ + .annotate( + tree_dataset=JSONObject( + id=F('id'), + label=F('name'), + isRoot=Case( + When( + Exists(OntologyTag.parents.through.objects.filter( + child_id=OuterRef('pk'), + )), + then=False + ), + default=True + ), + isLeaf=Case( + When(child_count__lt=1, then=True), + default=False + ), + type_id=F('type_id') + ) + ) \ + .values_list('tree_dataset', flat=True) + + nodes = [ + node | { 'full_names': roots.get(node.get('id')) } + if not node.get('isRoot') and roots.get(node.get('id')) else node + for node in nodes + ] + + return nodes + + @classmethod + def get_creation_data(cls, node_ids, type_ids, default=None): + """ + Attempts to derive the ontology data associated given a list of nodes + and their type_ids - will return the default value if it fails. + + This is a required step in preparing the creation data, since we + need to derive the path of each node so that we can merge it into the given + root node data. + + Args: + node_ids (int[]): a list of node ids + + type_ids (int): the ontology type ids + + default (any|None): the default return value + + Returns: + Either (a) the default value if we're unable to resolve the data, + or (b) a dict containing the value id(s) and any pre-fetched ancestor-related data + + """ + if not isinstance(node_ids, list) or not isinstance(type_ids, list): + return default + + node_ids = [int(node_id) for node_id in node_ids if gen_utils.parse_int(node_id, default=None) is not None] + type_ids = [int(type_id) for type_id in type_ids if gen_utils.parse_int(type_id, default=None) is not None] + + if len(node_ids) < 1 or len(type_ids) < 1: + return default + + nodes = OntologyTag.objects.filter(id__in=node_ids, type_id__in=type_ids) + ancestors = [ + [ + OntologyTag.get_node_data(ancestor.id, default=None) + for ancestor in node.ancestors() + ] + for node in nodes + if not node.is_root() and not node.is_island() + ] + + return { + 'ancestors': ancestors, + 'value': [OntologyTag.get_node_data(node_id) for node_id in node_ids], + } + + @classmethod + def get_detailed_source_value(cls, node_ids, type_ids, default=None): + """ + Attempts to format the ontology data in a similar to manner + that's composed via `template_utils.get_detailed_sourced_value()` + + Args: + node_ids (int[]): a list of node ids + + type_ids (int): the ontology type ids + + default (any|None): the default return value + + Returns: + Either (a) the default value if we're unable to resolve the data, + or (b) a list of objects containing the sourced value data + + """ + if not isinstance(node_ids, list) or not isinstance(type_ids, list): + return default + + node_ids = [int(node_id) for node_id in node_ids if gen_utils.parse_int(node_id, default=None) is not None] + type_ids = [int(type_id) for type_id in type_ids if gen_utils.parse_int(type_id, default=None) is not None] + + if len(node_ids) < 1 or len(type_ids) < 1: + return default + + nodes = OntologyTag.objects.filter(id__in=node_ids, type_id__in=type_ids) + if nodes.count() < 1: + return default + + return list(nodes.annotate(value=F('id')).values('name', 'value')) + + @transaction.atomic + def save(self, *args, **kwargs): + """ + Save override to apply validation or + modification methods dependent on the + associated `type_id` + """ + internal_type = self.type_id + if internal_type == constants.ONTOLOGY_TYPES.CLINICAL_DISEASE: + code_id = self.__validate_disease_code_id(self.properties) + if isinstance(code_id, int): + self.properties.update({ 'code_id': code_id }) + + super().save(*args, **kwargs) diff --git a/CodeListLibrary_project/clinicalcode/models/Template.py b/CodeListLibrary_project/clinicalcode/models/Template.py index 4f4f1cff4..c3b5513d1 100644 --- a/CodeListLibrary_project/clinicalcode/models/Template.py +++ b/CodeListLibrary_project/clinicalcode/models/Template.py @@ -3,8 +3,8 @@ from django.db import models from simple_history.models import HistoricalRecords -from .TimeStampedModel import TimeStampedModel from .EntityClass import EntityClass +from .TimeStampedModel import TimeStampedModel class Template(TimeStampedModel): ''' diff --git a/CodeListLibrary_project/clinicalcode/templatetags/detail_pg_renderer.py b/CodeListLibrary_project/clinicalcode/templatetags/detail_pg_renderer.py index 8909b1d0f..3ad56fe01 100644 --- a/CodeListLibrary_project/clinicalcode/templatetags/detail_pg_renderer.py +++ b/CodeListLibrary_project/clinicalcode/templatetags/detail_pg_renderer.py @@ -11,6 +11,7 @@ from ..entity_utils import permission_utils, template_utils, model_utils, gen_utils, constants, concept_utils from ..models.GenericEntity import GenericEntity +from ..models.OntologyTag import OntologyTag register = template.Library() @@ -244,16 +245,21 @@ def get_template_creation_data(entity, layout, field, request=None, default=None remove_userdata=True, hide_user_details=True, include_component_codes=False, - include_attributes=True, + include_attributes=True, + requested_entity_id=entity.id, include_reviewed_codes=True, derive_access_from=request ) - if value: values.append(value) return values + elif field_type == 'int_array': + source_info = validation.get('source') + tree_models = source_info.get('trees') if isinstance(source_info, dict) else None + if isinstance(tree_models, list): + return OntologyTag.get_detail_data(node_ids=data, default=default) if info.get('field_type') == 'data_sources': return get_data_sources(data, info, default=default) @@ -265,6 +271,8 @@ def get_template_creation_data(entity, layout, field, request=None, default=None class EntityWizardSections(template.Node): + SECTION_END = render_to_string(template_name=constants.DETAIL_WIZARD_SECTION_END) + def __init__(self, params, nodelist): self.request = template.Variable('request') self.params = params @@ -300,6 +308,11 @@ def __try_get_computed(self, request, field): if field == 'group': return permission_utils.get_user_groups(request) + def __append_section(self, output, section_content): + if gen_utils.is_empty_string(section_content): + return output + return output + section_content + self.SECTION_END + def __generate_wizard(self, request, context): output = '' template = context.get('template', None) @@ -426,8 +439,8 @@ def __generate_wizard(self, request, context): section_content += self.__try_render_item(template_name=uri, request=request, context=context.flatten() | { 'component': component }) if field_count > 0: - output += section_content - output += render_to_string(template_name=constants.DETAIL_WIZARD_SECTION_END, request=request, context=context.flatten() | { 'section': section }) + output = self.__append_section(output, section_content) + return output def render(self, context): diff --git a/CodeListLibrary_project/clinicalcode/templatetags/entity_renderer.py b/CodeListLibrary_project/clinicalcode/templatetags/entity_renderer.py index 5a28bb41c..37c06ed6e 100644 --- a/CodeListLibrary_project/clinicalcode/templatetags/entity_renderer.py +++ b/CodeListLibrary_project/clinicalcode/templatetags/entity_renderer.py @@ -3,6 +3,8 @@ from jinja2.exceptions import TemplateSyntaxError from django.template.loader import render_to_string from django.utils.translation import gettext_lazy as _ +from django.urls import reverse +from datetime import datetime import re import json @@ -69,6 +71,23 @@ def get_brand_base_embed_img(brand): return settings.APP_EMBED_ICON.format(logo_path=settings.APP_LOGO_PATH) return settings.APP_EMBED_ICON.format(logo_path=brand.logo_path) +@register.simple_tag +def render_citation_block(entity, request): + phenotype_id = f'{entity.id} / {entity.history_id}' + name = entity.name + author = entity.author + updated = entity.updated.strftime('%d %B %Y') + date = datetime.now().strftime('%d %B %Y') + url = request.build_absolute_uri(reverse( + 'entity_history_detail', + kwargs={ 'pk': entity.id, 'history_id': entity.history_id } + )) + + brand = request.BRAND_OBJECT + site_name = settings.APP_TITLE if not brand or not getattr(brand, 'site_title') else brand.site_title + + return f'{author}. *{phenotype_id} - {name}*. {site_name} [Online]. {updated}. Available from: [{url}]({url}). [Accessed {date}]' + @register.inclusion_tag('components/search/pagination/pagination.html', takes_context=True, name='render_entity_pagination') def render_pagination(context, *args, **kwargs): """ @@ -566,6 +585,8 @@ def render_steps_wizard(parser, token): return EntityWizardSections(params, nodelist) class EntityWizardSections(template.Node): + SECTION_END = render_to_string(template_name=constants.CREATE_WIZARD_SECTION_END) + def __init__(self, params, nodelist): self.request = template.Variable('request') self.params = params @@ -640,6 +661,11 @@ def __apply_mandatory_property(self, template, field): mandatory = template_utils.try_get_content(validation, 'mandatory') return mandatory if isinstance(mandatory, bool) else False + def __append_section(self, output, section_content): + if gen_utils.is_empty_string(section_content): + return output + return output + section_content + self.SECTION_END + def __generate_wizard(self, request, context): """ Generates the creation wizard template @@ -659,7 +685,7 @@ def __generate_wizard(self, request, context): sections.extend(constants.APPENDED_SECTIONS) for section in sections: - output += self.__try_render_item(template_name=constants.CREATE_WIZARD_SECTION_START, request=request, context=context.flatten() | { 'section': section }) + section_content = self.__try_render_item(template_name=constants.CREATE_WIZARD_SECTION_START, request=request, context=context.flatten() | { 'section': section }) for field in section.get('fields'): if template_utils.is_metadata(GenericEntity, field): @@ -720,9 +746,9 @@ def __generate_wizard(self, request, context): component['mandatory'] = self.__apply_mandatory_property(template_field, field) uri = f'{constants.CREATE_WIZARD_INPUT_DIR}/{component.get("input_type")}.html' - output += self.__try_render_item(template_name=uri, request=request, context=context.flatten() | { 'component': component }) + section_content += self.__try_render_item(template_name=uri, request=request, context=context.flatten() | { 'component': component }) + output = self.__append_section(output, section_content) - output += render_to_string(template_name=constants.CREATE_WIZARD_SECTION_END, request=request, context=context.flatten() | { 'section': section }) return output def render(self, context): diff --git a/CodeListLibrary_project/clinicalcode/urls.py b/CodeListLibrary_project/clinicalcode/urls.py index d00f15389..853278660 100644 --- a/CodeListLibrary_project/clinicalcode/urls.py +++ b/CodeListLibrary_project/clinicalcode/urls.py @@ -89,12 +89,11 @@ url(r'^phenotypes/(?P\w+)/(?P\d+)/submit/$', Publish.RequestPublish.as_view(),name='generic_entity_request_publish'), ] -# Add sitemaps & robots if required -if settings.IS_HDRUK_EXT == "1" or settings.IS_DEVELOPMENT_PC: - urlpatterns += [ - url(r'^robots.txt/$', site.robots_txt, name='robots.txt'), - url(r'^sitemap.xml/$', site.get_sitemap, name='sitemap.xml'), - ] +# Add sitemaps & robots if required (only for HDRUK .org site (or local dev.) -- check is done in site.py) +urlpatterns += [ + url(r'^robots.txt/$', site.robots_txt, name='robots.txt'), + url(r'^sitemap.xml/$', site.get_sitemap, name='sitemap.xml'), +] # Add non-readonly pages if not settings.CLL_READ_ONLY: @@ -118,5 +117,7 @@ # url(r'^adminTemp/admin_fix_breathe_dt/$', adminTemp.admin_fix_breathe_dt, name='admin_fix_breathe_dt'), #url(r'^adminTemp/admin_fix_malformed_codes/$', adminTemp.admin_fix_malformed_codes, name='admin_fix_malformed_codes'), url(r'^adminTemp/admin_force_adp_links/$', adminTemp.admin_force_adp_linkage, name='admin_force_adp_links'), + url(r'^adminTemp/admin_fix_coding_system_linkage/$', adminTemp.admin_fix_coding_system_linkage, name='admin_fix_coding_system_linkage'), + url(r'^adminTemp/admin_fix_concept_linkage/$', adminTemp.admin_fix_concept_linkage, name='admin_fix_concept_linkage'), url(r'^adminTemp/admin_force_brand_links/$', adminTemp.admin_force_brand_links, name='admin_force_brand_links'), ] diff --git a/CodeListLibrary_project/clinicalcode/views/GenericEntity.py b/CodeListLibrary_project/clinicalcode/views/GenericEntity.py index a87355134..419a836c4 100644 --- a/CodeListLibrary_project/clinicalcode/views/GenericEntity.py +++ b/CodeListLibrary_project/clinicalcode/views/GenericEntity.py @@ -8,7 +8,7 @@ from django.core.exceptions import BadRequest from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied -from django.http import HttpResponseNotFound +from django.http import HttpResponseNotFound, HttpResponseBadRequest from django.http.response import HttpResponse, JsonResponse, Http404 from django.shortcuts import render, redirect from django.template.loader import render_to_string @@ -290,8 +290,8 @@ def render_view(self, request, *args, **kwargs): template_id = gen_utils.parse_int(template_id, default=None) if template_id is not None: template = model_utils.try_get_instance(Template, pk=template_id) - if template is None: - raise BadRequest('Invalid request.') + if template is None or template.hide_on_create: + raise Http404 return self.create_form(request, context, template) # Send to update form if entity_id is selected @@ -299,16 +299,16 @@ def render_view(self, request, *args, **kwargs): if entity_id is not None and entity_history_id is not None: entity = create_utils.try_validate_entity(request, entity_id, entity_history_id) if not entity: - raise PermissionDenied + raise Http404 template = entity.template if template is None: - raise BadRequest('Invalid request.') + raise Http404 return self.update_form(request, context, template, entity) # Raise 400 if no param matches views - raise BadRequest('Invalid request.') + raise HttpResponseBadRequest('Invalid request.') ''' Forms ''' def select_form(self, request, context): diff --git a/CodeListLibrary_project/clinicalcode/views/View.py b/CodeListLibrary_project/clinicalcode/views/View.py index 4db3476c6..8aabc8d3d 100644 --- a/CodeListLibrary_project/clinicalcode/views/View.py +++ b/CodeListLibrary_project/clinicalcode/views/View.py @@ -15,12 +15,17 @@ from django.shortcuts import render from ..forms.ContactUsForm import ContactForm + +from ..models.Tag import Tag from ..models.Brand import Brand from ..models.CodingSystem import CodingSystem from ..models.DataSource import DataSource from ..models.Statistics import Statistics -from ..models.Tag import Tag -from ..entity_utils.permission_utils import redirect_readonly +from ..models.OntologyTag import OntologyTag + +from ..entity_utils.constants import ONTOLOGY_TYPES +from ..entity_utils.permission_utils import should_render_template, redirect_readonly + logger = logging.getLogger(__name__) @@ -289,7 +294,15 @@ def reference_data(request): 'data_sources': list(DataSource.objects.all().order_by('id').values('id', 'name')), 'coding_system': list(CodingSystem.objects.all().order_by('id').values('id', 'name')), 'tags': list(tags), - 'collections': list(collections) + 'collections': list(collections), } + # + # [!] Note: Temporary solution to block ontology rendering on reference data + # + # i.e. remove reference data to ontology unless template.hide_on_create=False + # + if should_render_template(name='Atlas Phecode Phenotype'): + context.update({ 'ontology': OntologyTag.get_groups([x.value for x in ONTOLOGY_TYPES], default=[]) }) + return render(request, 'clinicalcode/about/reference_data.html', context) diff --git a/CodeListLibrary_project/clinicalcode/views/adminTemp.py b/CodeListLibrary_project/clinicalcode/views/adminTemp.py index 3b17b4612..b4f11f20c 100644 --- a/CodeListLibrary_project/clinicalcode/views/adminTemp.py +++ b/CodeListLibrary_project/clinicalcode/views/adminTemp.py @@ -226,6 +226,257 @@ def admin_fix_malformed_codes(request): } ) +@login_required +def admin_fix_concept_linkage(request): + if settings.CLL_READ_ONLY: + raise PermissionDenied + + if not request.user.is_superuser: + raise PermissionDenied + + if not permission_utils.is_member(request.user, 'system developers'): + raise PermissionDenied + + # get + if request.method == 'GET': + return render( + request, + 'clinicalcode/adminTemp/admin_temp_tool.html', + { + 'url': reverse('admin_fix_concept_linkage'), + 'action_title': 'Fix Concept Linkage', + 'hide_phenotype_options': True, + } + ) + + # post + if request.method != 'POST': + raise BadRequest('Invalid') + + row_count = 0 + with connection.cursor() as cursor: + + ''' Update from legacy reference first ... ''' + sql = ''' + + with + legacy_reference as ( + select phenotype.id as phenotype_id, + concept->>'concept_id' as concept_id, + concept->>'concept_version_id' as concept_version_id, + created + from public.clinicalcode_phenotype as phenotype, + json_array_elements(phenotype.concept_informations::json) as concept + ), + ranked_legacy_concepts as ( + select phenotype_id, + cast(concept_id as integer) as concept_id, + cast(concept_version_id as integer) as concept_version_id, + rank() over ( + partition by phenotype_id + order by created asc + ) as ranking + from legacy_reference + ) + + update public.clinicalcode_historicalconcept as trg + set phenotype_owner_id = src.phenotype_id + from ( + select * + from ranked_legacy_concepts + where ranking = 1 + ) as src + where trg.id = src.concept_id + and trg.phenotype_owner_id is null; + + ''' + + cursor.execute(sql) + row_count += cursor.rowcount + + ''' ... then update from current reference ''' + sql = ''' + + with + entity_reference as ( + select id as phenotype_id, + history_id as phenotype_version_id, + cast(concepts->>'concept_id' as integer) as concept_id, + cast(concepts->>'concept_version_id' as integer) as concept_version_id + from ( + select id, + history_id, + concepts + from public.clinicalcode_historicalgenericentity as entity, + json_array_elements(entity.template_data::json->'concept_information') as concepts + where template_id = 1 + and json_array_length(entity.template_data::json->'concept_information') > 0 + ) hge_concepts + ), + first_child_concept as ( + select concept_id as concept_id, + min(concept_version_id) as concept_version_id + from entity_reference as entity + group by concept_id + ), + earliest_entity as ( + select phenotype_id, + concept_id, + concept_version_id + from ( + select phenotype_id, + rank() over ( + partition by phenotype_id + order by phenotype_version_id asc + ) as ranking, + concept.concept_id, + concept.concept_version_id + from entity_reference as entity + join first_child_concept as concept + using (concept_id, concept_version_id) + ) as hci + where ranking = 1 + ) + + update public.clinicalcode_historicalconcept as trg + set phenotype_owner_id = src.phenotype_id + from earliest_entity as src + where trg.id = src.concept_id + and trg.phenotype_owner_id is null; + + ''' + + cursor.execute(sql) + row_count += cursor.rowcount + + return render( + request, + 'clinicalcode/adminTemp/admin_temp_tool.html', + { + 'pk': -10, + 'rowsAffected' : { '1': f'Updated {str(row_count)} entities' }, + 'action_title': 'Fix Concept Linkage', + 'hide_phenotype_options': True, + } + ) + +@login_required +def admin_fix_coding_system_linkage(request): + if settings.CLL_READ_ONLY: + raise PermissionDenied + + if not request.user.is_superuser: + raise PermissionDenied + + if not permission_utils.is_member(request.user, 'system developers'): + raise PermissionDenied + + # get + if request.method == 'GET': + return render( + request, + 'clinicalcode/adminTemp/admin_temp_tool.html', + { + 'url': reverse('admin_fix_coding_system_linkage'), + 'action_title': 'Fix Coding System Linkage', + 'hide_phenotype_options': True, + } + ) + + # post + if request.method != 'POST': + raise BadRequest('Invalid') + + row_count = 0 + with connection.cursor() as cursor: + sql = ''' + + update public.clinicalcode_historicalgenericentity as trg + set template_data['coding_system'] = to_jsonb(src.coding_system) + from ( + select entity.phenotype_id, + entity.phenotype_version_id, + array_agg(distinct concept.coding_system_id::integer) as coding_system + from public.clinicalcode_historicalconcept as concept + join ( + select id as phenotype_id, + history_id as phenotype_version_id, + cast(concepts->>'concept_id' as integer) as concept_id, + cast(concepts->>'concept_version_id' as integer) as concept_version_id + from ( + select id, + history_id, + concepts + from public.clinicalcode_historicalgenericentity as entity, + json_array_elements(entity.template_data::json->'concept_information') as concepts + where template_id = 1 + and json_array_length(entity.template_data::json->'concept_information') > 0 + ) results + ) as entity + on entity.concept_id = concept.id + and entity.concept_version_id = concept.history_id + group by entity.phenotype_id, + entity.phenotype_version_id + ) src + where trg.id = src.phenotype_id + and trg.history_id = src.phenotype_version_id + and trg.template_id = 1 + and array( + select jsonb_array_elements_text(trg.template_data->'coding_system') + )::int[] <> src.coding_system; + + ''' + + cursor.execute(sql) + row_count = cursor.rowcount + + sql = ''' + + update public.clinicalcode_genericentity as trg + set template_data['coding_system'] = to_jsonb(src.coding_system) + from ( + select entity.phenotype_id, + array_agg(distinct concept.coding_system_id::integer) as coding_system + from public.clinicalcode_historicalconcept as concept + join ( + select id as phenotype_id, + cast(concepts->>'concept_id' as integer) as concept_id, + cast(concepts->>'concept_version_id' as integer) as concept_version_id + from ( + select id, + concepts + from public.clinicalcode_genericentity as entity, + json_array_elements(entity.template_data::json->'concept_information') as concepts + where template_id = 1 + and json_array_length(entity.template_data::json->'concept_information') > 0 + ) results + ) as entity + on entity.concept_id = concept.id + and entity.concept_version_id = concept.history_id + group by entity.phenotype_id + ) src + where trg.id = src.phenotype_id + and trg.template_id = 1 + and array( + select jsonb_array_elements_text(trg.template_data->'coding_system') + )::int[] <> src.coding_system; + + ''' + + cursor.execute(sql) + row_count += cursor.rowcount + + return render( + request, + 'clinicalcode/adminTemp/admin_temp_tool.html', + { + 'pk': -10, + 'rowsAffected' : { '1': f'Updated {str(row_count)} entities' }, + 'action_title': 'Fix Coding System Linkage', + 'hide_phenotype_options': True, + } + ) + @login_required def admin_force_adp_linkage(request): if settings.CLL_READ_ONLY: diff --git a/CodeListLibrary_project/clinicalcode/views/site.py b/CodeListLibrary_project/clinicalcode/views/site.py index 649be849c..777a358ca 100644 --- a/CodeListLibrary_project/clinicalcode/views/site.py +++ b/CodeListLibrary_project/clinicalcode/views/site.py @@ -5,11 +5,16 @@ from django.urls import reverse from clinicalcode.entity_utils import entity_db_utils +from django.core.exceptions import PermissionDenied cur_time = str(datetime.now().date()) @require_GET def robots_txt(request): + if not(settings.IS_HDRUK_EXT == "1" or settings.IS_DEVELOPMENT_PC): + raise PermissionDenied + + lines = [ "User-Agent: *", "Allow: /", @@ -28,6 +33,8 @@ def robots_txt(request): @require_GET def get_sitemap(request): + if not(settings.IS_HDRUK_EXT == "1" or settings.IS_DEVELOPMENT_PC): + raise PermissionDenied links = [ (request.build_absolute_uri(reverse('concept_library_home')), cur_time, "1.00"), diff --git a/CodeListLibrary_project/cll/static/js/clinicalcode/accessibility.js b/CodeListLibrary_project/cll/static/js/clinicalcode/accessibility.js index 2da624897..cdb3ceb1c 100644 --- a/CodeListLibrary_project/cll/static/js/clinicalcode/accessibility.js +++ b/CodeListLibrary_project/cll/static/js/clinicalcode/accessibility.js @@ -15,6 +15,17 @@ const CL_ACCESSIBILITY_KEYS = { * determine whether to initialise an interaction with that component */ domReady.finally(() => { + observeMatchingElements('[data-page]', (elem) => { + if (isNullOrUndefined(elem)) { + return; + } + + if (!elem.getAttribute('target') && !elem.getAttribute('href')) { + elem.setAttribute('target', '_blank'); + elem.setAttribute('href', '#'); + } + }); + document.addEventListener('keydown', e => { const elem = document.activeElement; const code = e.keyIdentifier || e.which || e.keyCode; @@ -22,7 +33,7 @@ domReady.finally(() => { return; } - if (elem.matches('[role="button"]')) { + if (elem.matches('[role="button"]') || elem.matches('[data-page]')) { elem.click(); } else if (elem.matches('[role="dropdown"]')) { const radio = elem.querySelector('input[type="radio"]'); diff --git a/CodeListLibrary_project/cll/static/js/clinicalcode/components/brandurls.js b/CodeListLibrary_project/cll/static/js/clinicalcode/components/brandurls.js index 9831df412..39a688364 100644 --- a/CodeListLibrary_project/cll/static/js/clinicalcode/components/brandurls.js +++ b/CodeListLibrary_project/cll/static/js/clinicalcode/components/brandurls.js @@ -1,3 +1,13 @@ +/** + * brandUrlsgen + * @desc ...? + * @param {*} all_brands + * @param {*} prod + * @param {*} element + * @param {*} old_root + * @param {*} path + * + */ const brandUrlsgen = (all_brands,prod,element,old_root,path) =>{ new_root = ""; if (element.getAttribute('value') != '') { @@ -52,6 +62,11 @@ const brandUrlsgen = (all_brands,prod,element,old_root,path) =>{ } } +/** + * generateOldPathRoot + * @desc ...? + * @returns {array} + */ const generateOldPathRoot = () =>{ var lTrimRegex = new RegExp("^/"); var lTrim = function (input) { diff --git a/CodeListLibrary_project/cll/static/js/clinicalcode/components/dropdown.js b/CodeListLibrary_project/cll/static/js/clinicalcode/components/dropdown.js index 5e4670689..f6ca04b26 100644 --- a/CodeListLibrary_project/cll/static/js/clinicalcode/components/dropdown.js +++ b/CodeListLibrary_project/cll/static/js/clinicalcode/components/dropdown.js @@ -14,6 +14,7 @@ const DROPDOWN_KEYS = { * createDropdownSelectionElement * @desc Creates an accessible dropdown element * @param {node} element the element to apply a dropdown element to + * */ const createDropdownSelectionElement = (element) => { const container = createElement('div', { diff --git a/CodeListLibrary_project/cll/static/js/clinicalcode/components/entityCreator.js b/CodeListLibrary_project/cll/static/js/clinicalcode/components/entityCreator.js new file mode 100644 index 000000000..508904204 --- /dev/null +++ b/CodeListLibrary_project/cll/static/js/clinicalcode/components/entityCreator.js @@ -0,0 +1,13 @@ +import EntityCreator, { collectFormData as collectEntityCreatorFormData } from '../forms/entityCreator/index.js'; + +/** + * Main thread + * @desc initialises the form after collecting the assoc. form data + */ +domReady.finally(() => { + const data = collectEntityCreatorFormData(); + + window.entityForm = new EntityCreator(data, { + promptUnsaved: true, + }); +}); diff --git a/CodeListLibrary_project/cll/static/js/clinicalcode/forms/entitySelector.js b/CodeListLibrary_project/cll/static/js/clinicalcode/components/entitySelector.js similarity index 76% rename from CodeListLibrary_project/cll/static/js/clinicalcode/forms/entitySelector.js rename to CodeListLibrary_project/cll/static/js/clinicalcode/components/entitySelector.js index 4941b621c..fb3e8e902 100644 --- a/CodeListLibrary_project/cll/static/js/clinicalcode/forms/entitySelector.js +++ b/CodeListLibrary_project/cll/static/js/clinicalcode/components/entitySelector.js @@ -1,21 +1,24 @@ /** - * Default description string if none provided + * ES_DEFAULT_DESCRIPTOR + * @desc Default description string if none provided + * */ -const DEFAULT_DESCRIPTOR = 'Create a ${name}' +const ES_DEFAULT_DESCRIPTOR = 'Create a ${name}' /** * getDescriptor * @desc gets the descriptor if valid, otherwise uses default format - * @param {string} description - * @param {string} name - * @returns {string} the descriptor + * @param {string} description the description associated with the item + * @param {string} name the name associated with the item + * @returns {string} the formatted descriptor + * */ const getDescriptor = (description, name) => { if (!isNullOrUndefined(description) && !isStringEmpty(description)) { return description; } - return new Function('name', `return \`${DEFAULT_DESCRIPTOR}\`;`)(name); + return new Function('name', `return \`${ES_DEFAULT_DESCRIPTOR}\`;`)(name); } /** @@ -23,15 +26,16 @@ const getDescriptor = (description, name) => { * @desc method to interpolate a card using a template * @param {node} container the container element * @param {string} template the template fragment - * @param {*} id any data value - * @param {*} title any data value - * @param {*} description any data value + * @param {number} id the id of the associated element + * @param {string} title string, as formatted by the `getDescriptor` method + * @param {string} description string, as formatted by the `getDescriptor` method * @returns {node} the interpolated element after appending to the container node + * */ const createGroup = (container, template, id, title, description) => { description = getDescriptor(description, title); - const html = interpolateHTML(template, { + const html = interpolateString(template, { 'id': id, 'title': title.toLocaleUpperCase(), 'description': description, @@ -46,15 +50,16 @@ const createGroup = (container, template, id, title, description) => { * @desc method to interpolate a card using a template * @param {node} container the container element * @param {string} template the template fragment - * @param {*} id any data value - * @param {*} type any data value - * @param {*} hint any data value - * @param {*} title any data value - * @param {*} description any data value + * @param {number} id the id of the associated element + * @param {any} type any data value relating to the type of the element + * @param {string} hint a string that defines the hint text of the element + * @param {string} title string, as formatted by the `getDescriptor` method + * @param {string} description string, as formatted by the `getDescriptor` method * @returns {node} the interpolated element after appending to the container node + * */ const createCard = (container, template, id, type, hint, title, description) => { - const html = interpolateHTML(template, { + const html = interpolateString(template, { 'type': type, 'id': id, 'hint': hint, @@ -71,6 +76,7 @@ const createCard = (container, template, id, type, hint, title, description) => * @desc Method that retrieves all relevant and