diff --git a/CURATION.md b/CURATION.md index e56cefc6..e84f837a 100644 --- a/CURATION.md +++ b/CURATION.md @@ -27,8 +27,12 @@ cd pkdb_data mkvirtualenv pkdb_data --python=python3.6 pip install -e . ``` -Next step is to export your credentials via environment variables. -Create a `.env` file with the following content +Next step is to export your credentials via environment variables. +Therefore, create a `.env` file by coping `env.template`. +``` +cp .env.template .env +``` +and update the the `USER` and `PASSWORD` in the `.env`. ``` API_BASE=https://develop.pk-db.com USER=PKDB_USERNAME @@ -51,7 +55,7 @@ To watch a given study use workon pkdb_data # export environment variables for backend -(pkdb_data) set -a && source .env.local +(pkdb_data) set -a && source .env # run the watch script (pkdb_data) watch_study -s $STUDYFOLDER @@ -98,8 +102,8 @@ contains all the relevant information "pkdb_version": 1.0, "name": "Author2007", "reference": 123456789, - "licence": "open || closed", - "access": "public || private", + "licence": "closed", + "access": "private", "creator": "mkoenig", "curators": [ ["mkoenig", 0.5] @@ -220,14 +224,16 @@ Individuals are curated very similar to groups with the exception that individua to a given group, i.e., the `group` attribute must be set. Individuals are most often defined based on spreadsheet mappings. See for instance below individuals which are defined via a table. +Information in the excel sheets can be referred to via using the `source` attribute stating the respective sheet (e.g. `Tab1` or `Fig2`). +The columns can then be matched via using the `col==col_header` syntax. In the example below for all the individuals the `name`, `group`, `age`, `weight` and `sex` are defined in the sheet `Tab1`. + ```json "individuals": [ { + "source": "Tab1", + "figure": "Tab1", "name": "col==subject", - "group": "col==group", - "source": "Akinyinka2000_Tab1.csv", - "format": "TSV", - "figure": "Akinyinka2000_Tab1.png", + "group": "col==group", "characteristica": [ { "measurement_type": "age", @@ -247,13 +253,13 @@ See for instance below individuals which are defined via a table. } ] ``` - +Even if individuals have no information on the characteristica, a table with individual names have to be created for later reference. ## 4. Curation of interventions/interventionset ```json { "interventionset": { - "description": "All patients and volunteers fasted overnight and, at 0800 hours, were given orally 300 mg caffeine dissolved in 150 ml water; food intake was allowed 3 h after administration of caffeine.", + "descriptions": [], "interventions": [ { "name": "glciv", @@ -280,7 +286,7 @@ All available fields for intervention and interventionset are: "substance": "categorial (substance)", "route": "categorial {oral, iv}", - "application": "categorial {'single dose', 'multiple doses', 'continuous injection'}", + "application": "categorial {'single dose', 'multiple dose', 'continuous injection'}", "form": "categorial {'tablete', 'capsule', ...}", "time": "double||double||double ...", "time_unit": "categorial", @@ -301,10 +307,10 @@ All available fields for intervention and interventionset are: ## 5. Curation of outputs and time courses The actual data in publication is available either from tables, figures or stated with the text. -All information should be curated by the means of excel spreadsheets, i.e., data must be digitized and transferred from the -PDF in a spreadsheet. +The information is curated either as excel spreadsheets (preferred method) or can be directly stored in the `study.json` (legacy method). The actual data is hereby digitized and stored in the spreadsheets. - Use Excel (LibreOffice/OpenOffice) spreadsheets to store digitized data +- store as `.xlsx` format (not `.ods`) - change language settings to use US numbers and formats, i.e. ‘.’ separator). Always use points (‘.’) as number separator, never comma (‘,’), i.e. 1.234 instead of 1,234. For all figures and tables from which data is extracted individual images (`png`) must be stored in the study folder, i.e., @@ -316,6 +322,8 @@ Use the screenshot functionality in the PDF viewer and save with image program l ### Figures - Use PlotDigitizer to digitize figures (https://sourceforge.net/projects/plotdigitizer/) + - download program + - run plotdigitizer with java - Open the image to digitize (`STUDYNAME_Fig[1-9]*.png`) - Use the Zoom function to increase the image if necessary (easier to click on data points) - First axes have to be calibrated (make sure to set logarithmical axes where necessary); calibration should be done very carfully because it will have a systematic effect (bias) on all digitized data points. @@ -332,13 +340,15 @@ Some tips for digitizion of figures: - set the number of digits to a reasonable value (2-3 digits) ### Tables +For tables the data is copied in spreadsheets. + +### Encoding outputs and time courses ```json { - "source": "Akinyinka2000_Tab3.csv", - "format": "TSV", - "subset": "substance==paraxanthine", - "figure": "Akinyinka2000_Tab3.png", + "source": "Tab3", + "figure": "Tab3", + "subset": "substance==paraxanthine", "group": "healthy subjects", "interventions": [ "Dcaf" @@ -356,12 +366,11 @@ Some tips for digitizion of figures: { "timecourses": [ { - "group": "all", + "source": "Fig1", + "figure": "Fig1", "groupby": "intervention", + "group": "all", "interventions": "col==intervention", - "source": "Albert1974_Fig1.tsv", - "format": "TSV", - "figure": "Albert1974_Fig1.png", "substance": "paracetamol", "tissue": "plasma", "measurement_type": "concentration", diff --git a/backend/pkdb_app/_version.py b/backend/pkdb_app/_version.py index 1e9fa4ea..79e30d3c 100644 --- a/backend/pkdb_app/_version.py +++ b/backend/pkdb_app/_version.py @@ -1,4 +1,4 @@ """ Definition of version string. """ -__version__ = "0.6.5" +__version__ = "0.6.7" diff --git a/backend/pkdb_app/behaviours.py b/backend/pkdb_app/behaviours.py index 3c6cc520..886b2d7f 100644 --- a/backend/pkdb_app/behaviours.py +++ b/backend/pkdb_app/behaviours.py @@ -15,9 +15,11 @@ class Meta: abstract = True class Externable(models.Model): - format = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + #format = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) subset_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) groupby = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + source_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + figure_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) class Meta: abstract = True diff --git a/backend/pkdb_app/categorials/behaviours.py b/backend/pkdb_app/categorials/behaviours.py index 62e0f25b..09feaa76 100644 --- a/backend/pkdb_app/categorials/behaviours.py +++ b/backend/pkdb_app/categorials/behaviours.py @@ -55,10 +55,10 @@ class Meta: class ExMeasurementTypeable(ValueableNotBlank,ValueableMapNotBlank): measurement_type = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - measurement_type_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + measurement_type_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) - choice = models.CharField(max_length=CHAR_MAX_LENGTH * 3, null=True) - choice_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + choice = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + choice_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) #substance = models.ForeignKey(Substance, null=True, on_delete=models.PROTECT) substance = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) substance_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) @@ -73,7 +73,7 @@ class MeasurementTypeable(ValueableNotBlank): measurement_type = models.ForeignKey(MeasurementType, on_delete=models.CASCADE) substance = models.ForeignKey(Substance, null=True, on_delete=models.PROTECT) - choice = models.CharField(max_length=CHAR_MAX_LENGTH * 3, null=True) + choice = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) class Meta: abstract = True diff --git a/backend/pkdb_app/categorials/models.py b/backend/pkdb_app/categorials/models.py index f2ea6d3c..f48da72b 100644 --- a/backend/pkdb_app/categorials/models.py +++ b/backend/pkdb_app/categorials/models.py @@ -12,7 +12,8 @@ from django.db import models from pkdb_app.categorials.managers import ChoiceManager from pkdb_app.users.models import User -from pkdb_app.utils import CHAR_MAX_LENGTH, create_choices, _validate_requried_key +from pkdb_app.utils import CHAR_MAX_LENGTH, create_choices, _validate_requried_key, CHAR_MAX_LENGTH_LONG + ureg = pint.UnitRegistry() # Units @@ -56,6 +57,7 @@ class Unit(models.Model): def p_unit(self): return ureg(self.name).u + class Annotation(models.Model): term = models.CharField(max_length=CHAR_MAX_LENGTH) relation = models.CharField(max_length=CHAR_MAX_LENGTH,) @@ -69,9 +71,35 @@ class Choice(models.Model): name = models.CharField(max_length=CHAR_MAX_LENGTH) annotations = models.ManyToManyField(Annotation) description = models.TextField(blank=True, null=True) + objects = ChoiceManager() - objects = ChoiceManager() +class Tissue(models.Model): + name = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + creator = models.ForeignKey(User, related_name="tissues", on_delete=models.CASCADE) + url_slug = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + description = models.TextField(blank=True, null=True) + + +class Route(models.Model): + name = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + creator = models.ForeignKey(User, related_name="routes", on_delete=models.CASCADE) + url_slug = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + description = models.TextField(blank=True, null=True) + + +class Application(models.Model): + name = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + creator = models.ForeignKey(User, related_name="applications", on_delete=models.CASCADE) + url_slug = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + description = models.TextField(blank=True, null=True) + + +class Form(models.Model): + name = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + creator = models.ForeignKey(User, related_name="forms", on_delete=models.CASCADE) + url_slug = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) + description = models.TextField(blank=True, null=True) class MeasurementType(models.Model): @@ -261,8 +289,8 @@ def validate_complete(self, data): if time_unit: self.validate_time_unit(time_unit) - time_requried_measurement_types = ["cumulative amount","cumulative metabolic ratio","recovery"] - if self.name in time_requried_measurement_types: + time_required_measurement_types = ["cumulative amount","cumulative metabolic ratio","recovery","auc_end"] + if self.name in time_required_measurement_types: details = f"for measurement type `{self.name}`" _validate_requried_key(data,"time", details=details) _validate_requried_key(data,"time_unit", details=details) diff --git a/backend/pkdb_app/categorials/serializers.py b/backend/pkdb_app/categorials/serializers.py index dcceb433..8e55fb41 100644 --- a/backend/pkdb_app/categorials/serializers.py +++ b/backend/pkdb_app/categorials/serializers.py @@ -5,7 +5,8 @@ # ---------------------------------- # Interventions # ---------------------------------- -from pkdb_app.categorials.models import Unit, Choice, MeasurementType, Annotation +from pkdb_app import utils +from pkdb_app.categorials.models import Unit, Choice, MeasurementType, Annotation, Tissue, Form, Application, Route from pkdb_app.serializers import WrongKeyValidationSerializer, ExSerializer from pkdb_app.substances.models import Substance from pkdb_app.utils import update_or_create_multiple @@ -19,7 +20,7 @@ class EXMeasurementTypeableSerializer(ExSerializer): class MeasurementTypeableSerializer(EXMeasurementTypeableSerializer): - substance = serializers.SlugRelatedField( + substance = utils.SlugRelatedField( slug_field="name", queryset=Substance.objects.all(), read_only=False, @@ -27,7 +28,7 @@ class MeasurementTypeableSerializer(EXMeasurementTypeableSerializer): allow_null=True, ) - measurement_type = serializers.SlugRelatedField( + measurement_type = utils.SlugRelatedField( slug_field="name", queryset=MeasurementType.objects.all()) @@ -50,6 +51,36 @@ class Meta: fields = ["name"] +class BaseCategorySerializer(WrongKeyValidationSerializer): + def to_internal_value(self, data): + self.validate_wrong_keys(data) + data["creator"] = self.context['request'].user.id + return super().to_internal_value(data) + + +class TissueSerializer(BaseCategorySerializer): + class Meta: + model = Tissue + fields = ["name","creator","url_slug", "description"] + + +class FormSerializer(BaseCategorySerializer): + class Meta: + model = Form + fields = ["name","creator","url_slug", "description"] + + +class ApplicationSerializer(BaseCategorySerializer): + class Meta: + model = Application + fields = ["name","creator","url_slug", "description"] + + +class RouteSerializer(BaseCategorySerializer): + class Meta: + model = Route + fields = ["name","creator","url_slug", "description"] + class AnnotationSerializer(serializers.ModelSerializer): term = serializers.CharField() description = serializers.CharField(allow_null=True) diff --git a/backend/pkdb_app/categorials/views.py b/backend/pkdb_app/categorials/views.py index 7c14c3cd..50c2f4c8 100644 --- a/backend/pkdb_app/categorials/views.py +++ b/backend/pkdb_app/categorials/views.py @@ -2,8 +2,9 @@ OrderingFilterBackend, CompoundSearchFilterBackend, MultiMatchSearchFilterBackend from django_elasticsearch_dsl_drf.viewsets import DocumentViewSet from pkdb_app.categorials.documents import MeasurementTypeDocument -from pkdb_app.categorials.models import MeasurementType -from pkdb_app.categorials.serializers import MeasurementTypeSerializer, MeasurementTypeElasticSerializer +from pkdb_app.categorials.models import MeasurementType, Tissue, Application, Route, Form +from pkdb_app.categorials.serializers import MeasurementTypeSerializer, MeasurementTypeElasticSerializer, \ + TissueSerializer, ApplicationSerializer, RouteSerializer, FormSerializer from pkdb_app.pagination import CustomPagination from pkdb_app.users.permissions import IsAdminOrCreator from rest_framework import viewsets @@ -15,6 +16,33 @@ class MeasurementTypeViewSet(viewsets.ModelViewSet): permission_classes = (IsAdminOrCreator,) lookup_field = "url_slug" + +class TissueViewSet(viewsets.ModelViewSet): + queryset = Tissue.objects.all() + serializer_class = TissueSerializer + permission_classes = (IsAdminOrCreator,) + lookup_field = "url_slug" + + +class ApplicationViewSet(viewsets.ModelViewSet): + queryset = Application.objects.all() + serializer_class = ApplicationSerializer + permission_classes = (IsAdminOrCreator,) + lookup_field = "url_slug" + + +class RouteViewSet(viewsets.ModelViewSet): + queryset = Route.objects.all() + serializer_class = RouteSerializer + permission_classes = (IsAdminOrCreator,) + lookup_field = "url_slug" + +class FormViewSet(viewsets.ModelViewSet): + queryset = Form.objects.all() + serializer_class = FormSerializer + permission_classes = (IsAdminOrCreator,) + lookup_field = "url_slug" + class MeasurementTypeElasticViewSet(DocumentViewSet): pagination_class = CustomPagination document = MeasurementTypeDocument diff --git a/backend/pkdb_app/interventions/documents.py b/backend/pkdb_app/interventions/documents.py index 262c2418..6b8fbb0e 100644 --- a/backend/pkdb_app/interventions/documents.py +++ b/backend/pkdb_app/interventions/documents.py @@ -15,14 +15,33 @@ class InterventionDocument(Document): 'raw': fields.StringField(analyzer='keyword'), } ) + + form = fields.StringField( + attr='form_name', + fields={ + 'raw': fields.StringField(analyzer='keyword'), + } + ) + route = fields.StringField( + attr='route_name', + fields={ + 'raw': fields.StringField(analyzer='keyword'), + } + ) + application = fields.StringField( + attr='application_name', + fields={ + 'raw': fields.StringField(analyzer='keyword'), + } + ) choice = string_field('choice') - application = string_field('application') + #application = string_field('application') time_unit = string_field('time_unit') time = fields.FloatField() substance = string_field('substance_name') study = string_field('study_name') - route = string_field('route') - form = string_field('form') + #route = string_field('route') + #form = string_field('form') name = string_field('name') normed = fields.BooleanField() raw = ObjectField(properties={ diff --git a/backend/pkdb_app/interventions/models.py b/backend/pkdb_app/interventions/models.py index 3d938db2..671ade9e 100644 --- a/backend/pkdb_app/interventions/models.py +++ b/backend/pkdb_app/interventions/models.py @@ -4,50 +4,19 @@ """ from django.db import models -from ..utils import CHAR_MAX_LENGTH, create_choices +from pkdb_app.categorials.models import Application, Form, Route +from ..utils import CHAR_MAX_LENGTH, CHAR_MAX_LENGTH_LONG from ..subjects.models import DataFile -from ..interventions.managers import InterventionSetManager,InterventionExManager +from ..interventions.managers import InterventionSetManager, InterventionExManager from ..behaviours import Externable, Accessible -from pkdb_app.users.models import User from pkdb_app.categorials.behaviours import Normalizable, ExMeasurementTypeable # ------------------------------------------------- # Intervention # ------------------------------------------------- - -# -# Choices for intervention routes, application and form. -# - -# FIXME: some duplication with pkdb_data -INTERVENTION_ROUTE = [ - "iv", # intravenous - "intramuscular", - "oral", - "rectal", - "inhalation" -] -INTERVENTION_APPLICATION = [ - "constant infusion", - "multiple dose", - "single dose", - "variable infusion", -] -INTERVENTION_FORM = [ - "capsule", - "tablet", - "solution", - "no info", -] - -INTERVENTION_APPLICATION_CHOICES = create_choices(INTERVENTION_APPLICATION) -INTERVENTION_ROUTE_CHOICES = create_choices(INTERVENTION_ROUTE) -INTERVENTION_FORM_CHOICES = create_choices(INTERVENTION_FORM) - - class InterventionSet(models.Model): objects = InterventionSetManager() @@ -72,13 +41,8 @@ def count(self): class AbstractIntervention(models.Model): - - - form = models.CharField(max_length=CHAR_MAX_LENGTH, null=True, choices=INTERVENTION_FORM_CHOICES) - application = models.CharField(max_length=CHAR_MAX_LENGTH, null=True, choices=INTERVENTION_APPLICATION_CHOICES) time = models.FloatField(null=True) time_unit = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - route = models.CharField(max_length=CHAR_MAX_LENGTH, null=True, choices=INTERVENTION_ROUTE_CHOICES) class Meta: abstract = True @@ -88,11 +52,11 @@ def __str__(self): class AbstractInterventionMap(models.Model): - form_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - application_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - time_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - time_unit_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - route_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + form_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + application_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + time_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + time_unit_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + route_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) class Meta: abstract = True @@ -103,7 +67,6 @@ class InterventionEx( AbstractIntervention, AbstractInterventionMap, ExMeasurementTypeable - ): """ Intervention (external curated layer).""" @@ -123,8 +86,13 @@ class InterventionEx( interventionset = models.ForeignKey( InterventionSet, related_name="intervention_exs", on_delete=models.CASCADE ) + + form = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + application = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + route = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + name = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - name_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + name_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) objects = InterventionExManager() class Meta: @@ -134,9 +102,10 @@ class Meta: class Intervention(Accessible, Normalizable, AbstractIntervention): """ A concrete step/thing which is done to the group. - In case of dosing/medication the actual dosing is stored in the Valueable. - In case of a step without dosing, e.g., lifestyle intervention only the measurement_type is used. - """ + In case of dosing/medication the actual dosing is stored in the Valueable. + In case of a step without dosing, e.g., lifestyle intervention only the + measurement_type is used. + """ ex = models.ForeignKey( InterventionEx, related_name="interventions", @@ -146,6 +115,30 @@ class Intervention(Accessible, Normalizable, AbstractIntervention): name = models.CharField(max_length=CHAR_MAX_LENGTH) + route = models.ForeignKey(Route, on_delete=models.CASCADE, null=True) + application = models.ForeignKey(Application, on_delete=models.CASCADE, null=True) + form = models.ForeignKey(Form, on_delete=models.CASCADE, null=True) + + @property + def route_name(self): + if self.route: + return self.route.name + else: + return None + + @property + def application_name(self): + if self.application: + return self.application.name + else: + return None + + @property + def form_name(self): + if self.form: + return self.form.name + else: + return None @property def study_name(self): @@ -154,5 +147,3 @@ def study_name(self): @property def study(self): return self.ex.interventionset.study - - diff --git a/backend/pkdb_app/interventions/serializers.py b/backend/pkdb_app/interventions/serializers.py index a552dc6c..3ce1aac6 100644 --- a/backend/pkdb_app/interventions/serializers.py +++ b/backend/pkdb_app/interventions/serializers.py @@ -3,8 +3,10 @@ """ import itertools +from pkdb_app import utils from pkdb_app.categorials.behaviours import VALUE_FIELDS, VALUE_FIELDS_NO_UNIT, \ MEASUREMENTTYPE_FIELDS, map_field, EX_MEASUREMENTTYPE_FIELDS +from pkdb_app.categorials.models import Tissue, Route, Application, Form from pkdb_app.categorials.serializers import MeasurementTypeableSerializer, EXMeasurementTypeableSerializer from pkdb_app.subjects.serializers import EXTERN_FILE_FIELDS from rest_framework import serializers @@ -61,9 +63,24 @@ class InterventionSerializer(MeasurementTypeableSerializer): + route = utils.SlugRelatedField( + slug_field="name", + required=False, + queryset=Route.objects.all()) + + application = utils.SlugRelatedField( + slug_field="name", + required=False, + queryset=Application.objects.all()) + + form = utils.SlugRelatedField( + slug_field="name", + required=False, + queryset=Form.objects.all()) + class Meta: model = Intervention - fields = INTERVENTION_FIELDS + MEASUREMENTTYPE_FIELDS + fields = INTERVENTION_FIELDS + MEASUREMENTTYPE_FIELDS def to_internal_value(self, data): @@ -139,6 +156,7 @@ def to_internal_value(self, data): # ---------------------------------- if not isinstance(data, dict): raise serializers.ValidationError(f"each intervention has to be a dict and not <{data}>") + temp_interventions = self.split_entry(data) for key in VALUE_FIELDS_NO_UNIT: if data.get(key) in NA_VALUES: @@ -231,6 +249,10 @@ class InterventionElasticSerializer(serializers.ModelSerializer): allowed_users = UserElasticSerializer(many=True, read_only=True) substance = serializers.CharField(allow_null=True) measurement_type = serializers.CharField() + route = serializers.CharField() + application = serializers.CharField() + form = serializers.CharField() + value = serializers.FloatField(allow_null=True) mean = serializers.FloatField(allow_null=True) median = serializers.FloatField(allow_null=True) diff --git a/backend/pkdb_app/interventions/views.py b/backend/pkdb_app/interventions/views.py index 3cd5cdd5..8597f248 100644 --- a/backend/pkdb_app/interventions/views.py +++ b/backend/pkdb_app/interventions/views.py @@ -1,9 +1,7 @@ from django.urls import reverse -from django_elasticsearch_dsl_drf.filter_backends import FilteringFilterBackend, CompoundSearchFilterBackend, \ +from django_elasticsearch_dsl_drf.filter_backends import FilteringFilterBackend, \ OrderingFilterBackend, IdsFilterBackend, MultiMatchSearchFilterBackend -from django_elasticsearch_dsl_drf.viewsets import DocumentViewSet -from pkdb_app.categorials.models import MeasurementType -from pkdb_app.interventions.models import INTERVENTION_ROUTE, INTERVENTION_FORM, INTERVENTION_APPLICATION +from pkdb_app.categorials.models import MeasurementType, Route, Form, Application from rest_framework import viewsets from rest_framework.response import Response @@ -28,9 +26,9 @@ def get_options(): options = {} options["measurement_type"] = {k.name: k._asdict() for k in MeasurementType.objects.all()} options["substances"] = reverse('substances_elastic-list') - options["route"] = INTERVENTION_ROUTE - options["form"] = INTERVENTION_FORM - options["application"] = INTERVENTION_APPLICATION + options["route"] = [k.name for k in Route.objects.all()] + options["form"] = [k.name for k in Form.objects.all()] + options["application"] = [k.name for k in Application.objects.all()] return options def list(self, request): @@ -49,7 +47,7 @@ class ElasticInterventionViewSet(AccessView): pagination_class = CustomPagination lookup_field = "id" filter_backends = [FilteringFilterBackend,IdsFilterBackend,OrderingFilterBackend,MultiMatchSearchFilterBackend] - search_fields = ('name','study','access','measurement_type','substance',"form","application",'route','time_unit') + search_fields = ('name','study','access','measurement_type','substance',"form","tissue","application",'route','time_unit') multi_match_search_fields = {field: {"boost": 1} for field in search_fields} multi_match_options = { 'operator': 'and' diff --git a/backend/pkdb_app/outputs/documents.py b/backend/pkdb_app/outputs/documents.py index ad82191c..b5cc773e 100644 --- a/backend/pkdb_app/outputs/documents.py +++ b/backend/pkdb_app/outputs/documents.py @@ -45,7 +45,7 @@ class OutputDocument(Document): unit = string_field('unit') time_unit = string_field('time_unit') time = fields.FloatField('null_time') - tissue = string_field('tissue') + tissue = string_field('tissue_name') measurement_type = string_field("measurement_type_name") access = string_field('access') allowed_users = fields.ObjectField( @@ -117,7 +117,7 @@ class TimecourseDocument(Document): time_unit = string_field('time_unit') figure = string_field('figure') time = fields.FloatField('null_time', multi=True) - tissue = string_field('tissue') + tissue = string_field('tissue_name') measurement_type = string_field("measurement_type_name") access = string_field('access') allowed_users = fields.ObjectField( diff --git a/backend/pkdb_app/outputs/models.py b/backend/pkdb_app/outputs/models.py index 7045f2e6..c459fa44 100644 --- a/backend/pkdb_app/outputs/models.py +++ b/backend/pkdb_app/outputs/models.py @@ -9,10 +9,10 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db import models from django.contrib.postgres.fields import ArrayField -from pkdb_app.categorials.models import MeasurementType, ureg +from pkdb_app.categorials.models import MeasurementType, ureg, Tissue from pkdb_app.interventions.models import Intervention -from ..utils import CHAR_MAX_LENGTH, create_choices +from ..utils import CHAR_MAX_LENGTH, CHAR_MAX_LENGTH_LONG from pkdb_app.subjects.models import Group, DataFile, Individual from .managers import ( @@ -25,21 +25,10 @@ from ..behaviours import ( Externable, Accessible) -from pkdb_app.categorials.behaviours import Normalizable, ExMeasurementTypeable, ValueableMapNotBlank, \ - ValueableNotBlank, MeasurementTypeable +from pkdb_app.categorials.behaviours import Normalizable, ExMeasurementTypeable TIME_NORM_UNIT = "hr" -OUTPUT_TISSUE_DATA = [ - "plasma", - "saliva", - "serum", - "spinal fluid", - "urine", - "breath", - "bile duct" -] -OUTPUT_TISSUE_DATA_CHOICES = create_choices(OUTPUT_TISSUE_DATA) # ------------------------------------------------- # OUTPUTS @@ -84,7 +73,6 @@ def count_timecourses(self): class AbstractOutput(models.Model): - tissue = models.CharField( max_length=CHAR_MAX_LENGTH, choices=OUTPUT_TISSUE_DATA_CHOICES, null=True) time = models.FloatField(null=True) time_unit = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) class Meta: @@ -92,9 +80,9 @@ class Meta: class AbstractOutputMap(models.Model): - tissue_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - time_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - time_unit_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + tissue_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + time_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + time_unit_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) class Meta: abstract = True @@ -115,26 +103,30 @@ class OutputEx(Externable, OutputSet, related_name="output_exs", on_delete=models.CASCADE, null=True ) - group = models.ForeignKey(Group, null=True, on_delete=models.CASCADE) individual = models.ForeignKey(Individual, null=True, on_delete=models.CASCADE) interventions = models.ManyToManyField(Intervention) - group_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - individual_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - interventions_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + group_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + individual_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + interventions_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) - objects = OutputExManager() + tissue = models.CharField( max_length=CHAR_MAX_LENGTH, null=True) + objects = OutputExManager() + + class Output(AbstractOutput,Normalizable, Accessible): """ Storage of data sets. """ group = models.ForeignKey(Group, null=True, blank=True, on_delete=models.CASCADE) individual = models.ForeignKey(Individual, null=True, blank=True, on_delete=models.CASCADE) _interventions = models.ManyToManyField(Intervention) - tissue = models.CharField(max_length=CHAR_MAX_LENGTH, choices=OUTPUT_TISSUE_DATA_CHOICES) + + tissue = models.ForeignKey(Tissue,related_name="outputs", null=True, blank=True, on_delete=models.CASCADE) + ex = models.ForeignKey(OutputEx, related_name="outputs", on_delete=models.CASCADE, null=True) # calculated by timecourse data @@ -142,6 +134,10 @@ class Output(AbstractOutput,Normalizable, Accessible): timecourse = models.ForeignKey("Timecourse", on_delete=models.CASCADE, related_name="pharmacokinetics", null=True) objects = OutputManager() + @property + def tissue_name(self): + return self.tissue.name + @property def interventions(self): interventions = self._interventions @@ -255,14 +251,16 @@ class TimecourseEx( individual = models.ForeignKey(Individual, null=True, on_delete=models.CASCADE) interventions = models.ManyToManyField(Intervention) - group_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - individual_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - interventions_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + group_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + individual_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + interventions_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + + tissue = models.CharField( max_length=CHAR_MAX_LENGTH, null=True) objects = TimecourseExManager() -class Timecourse(AbstractOutput, Normalizable,Accessible): +class Timecourse(AbstractOutput, Normalizable, Accessible): """ Storing of time course data. Store a binary blop of the data (json, pandas dataframe or similar, backwards compatible). @@ -273,7 +271,9 @@ class Timecourse(AbstractOutput, Normalizable,Accessible): ex = models.ForeignKey( TimecourseEx, related_name="timecourses", on_delete=models.CASCADE ) - tissue = models.CharField(max_length=CHAR_MAX_LENGTH, choices=OUTPUT_TISSUE_DATA_CHOICES) + #tissue = models.CharField(max_length=CHAR_MAX_LENGTH, choices=OUTPUT_TISSUE_DATA_CHOICES) + tissue = models.ForeignKey(Tissue,related_name="timecourses", null=True, blank=True, on_delete=models.CASCADE) + value = ArrayField(models.FloatField(null=True), null=True) mean = ArrayField(models.FloatField(null=True), null=True) @@ -286,7 +286,9 @@ class Timecourse(AbstractOutput, Normalizable,Accessible): time = ArrayField(models.FloatField(null=True), null=True) objects = OutputManager() - + @property + def tissue_name(self): + return self.tissue.name @property def interventions(self): diff --git a/backend/pkdb_app/outputs/serializers.py b/backend/pkdb_app/outputs/serializers.py index 50e00fd5..521565a9 100644 --- a/backend/pkdb_app/outputs/serializers.py +++ b/backend/pkdb_app/outputs/serializers.py @@ -2,13 +2,15 @@ Serializers for interventions. """ import numpy as np + +from pkdb_app import utils from pkdb_app.categorials.behaviours import MEASUREMENTTYPE_FIELDS, EX_MEASUREMENTTYPE_FIELDS, VALUE_FIELDS, \ VALUE_FIELDS_NO_UNIT, map_field from pkdb_app.categorials.serializers import MeasurementTypeableSerializer from pkdb_app.interventions.serializers import InterventionSmallElasticSerializer from rest_framework import serializers -from pkdb_app.categorials.models import MeasurementType +from pkdb_app.categorials.models import MeasurementType, Tissue from pkdb_app.users.serializers import UserElasticSerializer from ..comments.serializers import DescriptionSerializer, CommentSerializer, DescriptionElasticSerializer, \ @@ -65,6 +67,12 @@ class OutputSerializer(MeasurementTypeableSerializer): required=False, allow_null=True, ) + tissue = utils.SlugRelatedField( + slug_field="name", + queryset=Tissue.objects.all(), + read_only=False, + required=False + ) @@ -220,7 +228,7 @@ class TimecourseSerializer(BaseOutputExSerializer): read_only=False, required=False, ) - substance = serializers.SlugRelatedField( + substance = utils.SlugRelatedField( slug_field="name", queryset=Substance.objects.all(), read_only=False, @@ -228,12 +236,18 @@ class TimecourseSerializer(BaseOutputExSerializer): allow_null=True, ) - measurement_type = serializers.SlugRelatedField( + measurement_type = utils.SlugRelatedField( slug_field="name", queryset=MeasurementType.objects.all(), read_only=False, required=False ) + tissue = utils.SlugRelatedField( + slug_field="name", + queryset=Tissue.objects.all(), + read_only=False, + required=False + ) class Meta: @@ -324,6 +338,8 @@ def to_internal_value(self, data): # ---------------------------------- # decompress external format # ---------------------------------- + if not isinstance(data, dict): + raise serializers.ValidationError(f"each timecourse has to be a dict and not <{data}>") temp_timecourses = self.split_entry(data) timecourses = [] @@ -406,6 +422,7 @@ class OutputElasticSerializer(serializers.HyperlinkedModelSerializer): interventions = InterventionSmallElasticSerializer(many=True) substance = serializers.CharField() measurement_type = serializers.CharField() + tissue = serializers.CharField() value = serializers.FloatField(allow_null=True) mean = serializers.FloatField(allow_null=True) @@ -448,6 +465,8 @@ class TimecourseElasticSerializer(serializers.HyperlinkedModelSerializer): individual = IndividualSmallElasticSerializer() interventions = InterventionSmallElasticSerializer(many=True) measurement_type = serializers.CharField() + tissue = serializers.CharField() + raw = PkSerializer() pharmacokinetics = PkSerializer(many=True) substance = serializers.CharField() diff --git a/backend/pkdb_app/outputs/views.py b/backend/pkdb_app/outputs/views.py index 4332e38a..ff306068 100644 --- a/backend/pkdb_app/outputs/views.py +++ b/backend/pkdb_app/outputs/views.py @@ -1,10 +1,8 @@ from django.urls import reverse from django_elasticsearch_dsl_drf.constants import LOOKUP_QUERY_IN -from django_elasticsearch_dsl_drf.filter_backends import FilteringFilterBackend, CompoundSearchFilterBackend, \ +from django_elasticsearch_dsl_drf.filter_backends import FilteringFilterBackend, \ OrderingFilterBackend, IdsFilterBackend, MultiMatchSearchFilterBackend -from django_elasticsearch_dsl_drf.viewsets import DocumentViewSet -from pkdb_app.categorials.models import MeasurementType -from pkdb_app.outputs.models import OUTPUT_TISSUE_DATA +from pkdb_app.categorials.models import MeasurementType, Tissue from rest_framework import viewsets from rest_framework.response import Response @@ -25,7 +23,8 @@ def get_options(): options = {} options["measurememnt_types"] = {k.name: k._asdict() for k in MeasurementType.objects.all()} options["substances"] = reverse('substances_elastic-list') - options["tissue"] = OUTPUT_TISSUE_DATA + options["tissue"] = [k.name for k in Tissue.objects.all()] + return options def list(self, request): @@ -38,7 +37,7 @@ def get_options(): options = {} options["measurememnt_types"] = {k.name: k._asdict() for k in MeasurementType.objects.all()} options["substances"] = reverse('substances_elastic-list') - options["tissue"] = OUTPUT_TISSUE_DATA + options["tissue"] = [k.name for k in Tissue.objects.all()] return options def list(self, request): diff --git a/backend/pkdb_app/serializers.py b/backend/pkdb_app/serializers.py index 87eb991d..3d9aa179 100644 --- a/backend/pkdb_app/serializers.py +++ b/backend/pkdb_app/serializers.py @@ -1,12 +1,15 @@ import copy +from pathlib import Path import numpy as np import numbers import pandas as pd + from django.db.models import Q + from pkdb_app.categorials.behaviours import map_field from rest_framework import serializers from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from collections import OrderedDict, namedtuple +from collections import OrderedDict from rest_framework.settings import api_settings from pkdb_app.interventions.models import DataFile, Intervention from pkdb_app.normalization import get_se, get_sd, get_cv @@ -15,24 +18,11 @@ ITEM_SEPARATOR = "||" ITEM_MAPPER = "==" -NA_VALUES = ["na","NA","nan","NAN"] - +NA_VALUES = ["na", "NA", "nan", "NAN"] -# --------------------------------------------------- -# File formats -# --------------------------------------------------- -FileFormat = namedtuple("FileFormat", ["name", "delimiter"]) -FORMAT_MAPPING = {"TSV": FileFormat("TSV", "\t"), "CSV": FileFormat("CSV", ",")} - -# --------------------------------------------------- -# -# --------------------------------------------------- class WrongKeyValidationSerializer(serializers.ModelSerializer): - # ---------------------------------- - # helper - # ---------------------------------- @staticmethod def retransform_map_string(k): if "_map" in k: @@ -48,7 +38,9 @@ def validate_wrong_keys(self, data): for payload_key in payload_keys: if payload_key not in serializer_fields: payload_key = self.retransform_map_string(payload_key) - msg = {payload_key: f"`{payload_key}` is a wrong field, allowed fields are {[f for f in serializer_fields if not 'map' in f]}"} + msg = { + payload_key: f"'{payload_key}' is an incorrect field, " + f"supported fields are {sorted([f for f in serializer_fields if not 'map' in f])}"} raise serializers.ValidationError(msg) def get_or_val_error(self, model, *args, **kwargs): @@ -60,7 +52,7 @@ def get_or_val_error(self, model, *args, **kwargs): if not instance: raise serializers.ValidationError( { - api_settings.NON_FIELD_ERRORS_KEY: "instance does not exist", + api_settings.NON_FIELD_ERRORS_KEY: "instance does not exist.", "detail": {**kwargs}, } ) @@ -100,7 +92,8 @@ class MappingSerializer(WrongKeyValidationSerializer): @staticmethod def transform_map_fields(data): """ - replaces key with f"{key}_map" if value contains special syntax.( ==, || ) + replaces key with f"{key}_map" if value contains special syntax. + ( ==, || ) """ transformed_data = {} for key, value in data.items(): @@ -118,8 +111,6 @@ def retransform_map_fields(self,data): transformed_data = {} for k, v in data.items(): k = self.retransform_map_string(k) - #if v is None: - # continue transformed_data[k] = v return transformed_data @@ -140,8 +131,10 @@ def number_of_entries(self, entry): """ Splits the data to get number of entries. """ n_values = [] + for field, value in entry.items(): n = 1 + try: values = value.split(ITEM_SEPARATOR) n = len(values) @@ -172,15 +165,12 @@ def interventions_from_string(value): else: return value - def split_entry(self, entry): """ Splits entry fields based on separator. :param entry: :return: list of entries """ - - n = self.number_of_entries(entry) if n == 1: return [entry] @@ -201,20 +191,18 @@ def split_entry(self, entry): if field == "interventions": values[k] = self.interventions_from_string(value) - if values[k] in ["NA", "NAN", "na", "nan"]: + if values[k] in NA_VALUES: values[k] = None elif values[k] == "[]": values[k] = [] - - - # --- validation --- # names must be split in a split entry if field == "name" and len(values) != n: if len(values) == 1: raise serializers.ValidationError( - f"names have to be splitted and not left as <{values}>. Otherwise UniqueConstrain of name is violated." + f"names must be split and not left as <{values}>. " + f"Otherwise UniqueConstrain of name is violated." ) # check for old syntax @@ -225,7 +213,6 @@ def split_entry(self, entry): raise serializers.ValidationError( f"Splitting via '{{ }}' syntax not allowed, use '||' in count." ) - # ------------------ # extend entries if len(values) == 1: @@ -237,9 +224,6 @@ def split_entry(self, entry): ) for k in range(n): - # if field in ["count"]: - # entries[k][field] = int(values[k]) - # else: entries[k][field] = values[k] return entries @@ -275,47 +259,58 @@ def subset_pd(self, subset, df): ) return df + def df_from_file(self, source, subset): + """Creates dataframe from source and subset. - - def df_from_file(self, source, format, subset): - delimiter = FORMAT_MAPPING[format].delimiter - - - if isinstance(source,int): + :param source: + :param subset: + :return: pandas DataFrame + """ + if isinstance(source, int): pass - - elif isinstance(source,str): + elif isinstance(source, str): if source.isnumeric(): pass else: - - raise serializers.ValidationError( - {"source": f"<{str(source)}> is not existing", "detail": type(source)}) - + { + "source": f"<{str(source)}> does not exist", + "detail": type(source) + }) else: raise serializers.ValidationError( - {"source":f"<{str(source)}> is not existing","detail":type(source)}) + { + "source": f"<{str(source)}> does not exist", + "detail": type(source) + }) src = DataFile.objects.get(pk=source) + + if Path(src.file.name).suffix != ".tsv": + raise serializers.ValidationError( + { + "source": f"<{Path(src.file.name).name}> must be a TSV " + f"file with the suffix: <.tsv>" + }) + + # read the TSV try: df = pd.read_csv( src.file, - delimiter=delimiter, + delimiter="\t", keep_default_na=False, - na_values=["NA", "NAN", "na", "nan"], + na_values=NA_VALUES, ) - df.columns = df.columns.str.strip() - - except Exception as e: raise serializers.ValidationError( { - "source": "cannot read csv", - "detail": {"source": source, "format": format, "subset": subset}, + "source": "cannot read tsv", + "detail": {"source": source, "subset": subset}, } ) + + # filter subset if subset: if "&" in subset: for subset_single in [s.strip() for s in subset.split("&")]: @@ -323,14 +318,12 @@ def df_from_file(self, source, format, subset): else: df = self.subset_pd(subset, df) - return df - def make_entry(self, entry, template,data, source): + def make_entry(self, entry, template, data, source): entry_dict = copy.deepcopy(template) recursive_entry_dict = list(recursive_iter(entry_dict)) - for keys, value in recursive_entry_dict: if isinstance(value, str): if ITEM_MAPPER in value: @@ -347,9 +340,8 @@ def make_entry(self, entry, template,data, source): except AttributeError: raise serializers.ValidationError( - [ - f"key <{values[1]}> is missing in file <{DataFile.objects.get(pk=source).file}> ", + f"header key <{values[1]}> is missing in file <{DataFile.objects.get(pk=source).file}> ", data ] ) @@ -375,17 +367,10 @@ def entries_from_file(self, data): # get data template.pop("source", None) template.pop("figure", None) - format = template.pop("format", None) subset = template.pop("subset", None) if source: - - if format is None: - raise serializers.ValidationError({"format": "format is missing!"}) - df = self.df_from_file(source, format, subset) - - - + df = self.df_from_file(source, subset) template = copy.deepcopy(template) mappings = [] @@ -395,12 +380,15 @@ def entries_from_file(self, data): mappings.append(value) if len(mappings) == 0: - raise serializers.ValidationError({"source": "source is provided but the mapping operator == is not used in any field"}) + raise serializers.ValidationError( + {"source": "Source is provided but the mapping operator " + "'==' is not used in any field"}) if data.get("groupby"): groupby = template.pop("groupby") if not isinstance(groupby, str): - raise serializers.ValidationError({"groupby": "groupby has to be a string"}) + raise serializers.ValidationError( + {"groupby": "groupby must be a string"}) groupby = [v.strip() for v in groupby.split("&")] try: @@ -408,13 +396,11 @@ def entries_from_file(self, data): for entry in group_df.itertuples(): entry_dict = self.make_entry(entry, template, data, source) entries.append(entry_dict) - except KeyError: - raise serializers.ValidationError( - [ - f"Some keys in groupby <{groupby}> are missing in file <{DataFile.objects.get(pk=source).file}> ", + f"Some keys in groupby <{groupby}> are missing in " + f"file <{DataFile.objects.get(pk=source).file}> ", data ] ) @@ -428,38 +414,45 @@ def entries_from_file(self, data): return entries - def array_from_file(self, data): """ Handle conversion of time course data. :param data: :return: """ - source = data.get("source") if source: template = copy.deepcopy(data) + # get data template.pop("source") template.pop("figure", None) - format = template.pop("format", None) - if format is None: - raise serializers.ValidationError({"format": "format is missing!"}) subset = template.pop("subset", None) + # read dataframe subset - df = self.df_from_file(source, format, subset) + df = self.df_from_file(source, subset) template = copy.deepcopy(template) if data.get("groupby"): groupby = template.pop("groupby") if not isinstance(groupby, str): - raise serializers.ValidationError({"groupby":"groupby has to be a string"}) + raise serializers.ValidationError( + {"groupby": "groupby must be a string"} + ) groupby = [v.strip() for v in groupby.split("&")] array_dicts = [] try: df[groupby] except KeyError: - raise serializers.ValidationError({"groupby":f"keys <{groupby}> used for groupby are missing in source file <{DataFile.objects.get(pk=source).file.name}>. To group by more then one column you can use the & operator e.g. 'col1 & col2 & col3'"}) + raise serializers.ValidationError( + { + "groupby": + f"keys <{groupby}> used for groupby are " + f"missing in source file " + f"<{DataFile.objects.get(pk=source).file.name}>. " + f"To group by more then one column the '&' " + f"operator can be used. E.g. 'col1 & col2 & col3'" + }) for group_name, group_df in df.groupby(groupby): array_dict = copy.deepcopy(template) @@ -474,12 +467,11 @@ def array_from_file(self, data): else: raise serializers.ValidationError( - "For timecourse data a source file has to be provided." + "For timecourse data a 'source' file must be provided." ) return array_dicts - - def dict_from_array(self,array_dict,df,data,source): + def dict_from_array(self, array_dict, df, data, source): recursive_array_dict = list(recursive_iter(array_dict)) for keys, value in recursive_array_dict: @@ -496,41 +488,35 @@ def dict_from_array(self,array_dict,df,data,source): try: value_array = df[values[1]] - - except KeyError: raise serializers.ValidationError( [ - f"key <{values[1]}> is missing in file <{DataFile.objects.get(pk=source).file}> ", + f"header key <{values[1]}> is missing in file " + f"<{DataFile.objects.get(pk=source).file}>", data, ] ) - #to get rid of dict - if keys[0] in ["individual", "group","interventions","substance","tissue","time_unit","unit","measurement_type"]: - unqiue_values = value_array.unique() - if len(unqiue_values) != 1: + # get rid of dict + if keys[0] in ["individual", "group", "interventions", + "substance", "tissue", "time_unit", "unit", + "measurement_type"]: + unique_values = value_array.unique() + if len(unique_values) != 1: raise serializers.ValidationError( - [ - f"{values[1]} has to be a unique for one timecourse. <{unqiue_values}>", - data, - ]) + [f"{values[1]} has to be unique for one " + f"timecourse: <{unique_values}>", + data] + ) if keys[0] == "interventions": - entry_value = self.interventions_from_string(unqiue_values[0]) + entry_value = self.interventions_from_string(unique_values[0]) set_keys(array_dict, entry_value, *keys[:1]) - #array_dict[keys[0]] = [v.strip() for v in unqiue_values[0].split(",")] else: - set_keys(array_dict, unqiue_values[0], *keys) - + set_keys(array_dict, unique_values[0], *keys) else: set_keys(array_dict, value_array.values.tolist(), *keys) - - # ---------------------------------- - # - # ---------------------------------- - def to_internal_value(self, data): data = self.transform_map_fields(data) return super().to_internal_value(data) @@ -544,32 +530,35 @@ def to_representation(self, instance): # url representation of file for file in ["source", "figure"]: if file in rep: - rep[file] = request.build_absolute_uri(getattr(instance, file).file.url) + if "||" not in str(rep[file]): + rep[file] = request.build_absolute_uri(getattr(instance, file).file.url) + return rep class ExSerializer(MappingSerializer): - def to_internal_related_fields(self, data): study_sid = self.context["request"].path.split("/")[-2] if "group" in data: - if data["group"]: try: data["group"] = Group.objects.get( Q(ex__groupset__study__sid=study_sid) & Q(name=data.get("group")) ).pk - except ObjectDoesNotExist: - msg = f'group: {data.get("group")} in study: {study_sid} does not exist' + + except (ObjectDoesNotExist, MultipleObjectsReturned) as err: + if err == ObjectDoesNotExist: + msg = f'group: {data.get("group")} in study: {study_sid} does not exist' + else: + msg = f'group: {data.get("group")} in study: {study_sid} has been defined multiple times.' + raise serializers.ValidationError(msg) if "individual" in data: - if data["individual"]: - try: study_individuals = Individual.objects.filter( ex__individualset__study__sid=study_sid @@ -606,15 +595,14 @@ def to_internal_related_fields(self, data): data["interventions"] = interventions return data - ########################## - # helpers - ########################## - def _validate_figure(self,datafile): + @staticmethod + def _validate_figure(datafile): if datafile: - allowed_endings = ['png','jpg','jpeg','tif','tiff'] - if not any([datafile.file.name.endswith(ending)for ending in allowed_endings]): - raise serializers.ValidationError({"figure":f"{datafile.file.name} has to end with {allowed_endings}"}) - + allowed_endings = ['png', 'jpg', 'jpeg', 'tif', 'tiff'] + if not any([datafile.file.name.endswith(ending) + for ending in allowed_endings]): + raise serializers.ValidationError( + {"figure": f"{datafile.file.name} must end with {allowed_endings}"}) def _validate_disabled_data(self, data_dict, disabled): disabled = set(disabled) @@ -623,7 +611,10 @@ def _validate_disabled_data(self, data_dict, disabled): wrong_keys = self._retransform_map_list(wrong_keys) raise serializers.ValidationError( { - api_settings.NON_FIELD_ERRORS_KEY: f"The following keys are not allowed {wrong_keys} due to restricted keys on indivdual or group.", + api_settings.NON_FIELD_ERRORS_KEY: + f"The following keys are not allowed due to " + f"restricted keys on individual or group: " + f"'{wrong_keys}'", "detail": data_dict, } ) @@ -642,12 +633,10 @@ def _validate_group_output(self, data): disabled = ["value", "value_map"] self._validate_disabled_data(data, disabled) - def validate_group_individual_output(self, output): + @staticmethod + def validate_group_individual_output(output): is_group = output.get("group") or output.get("group_map") is_individual = output.get("individual") or output.get("individual_map") - print("*" * 100) - print(is_group,is_individual) - print("*" * 100) if is_individual and is_group: raise serializers.ValidationError( { @@ -656,7 +645,6 @@ def validate_group_individual_output(self, output): } ) - elif not (is_individual or is_group): raise serializers.ValidationError( { @@ -664,7 +652,8 @@ def validate_group_individual_output(self, output): } ) - def _key_is(self, data, key): + @staticmethod + def _key_is(data, key): return data.get(key) or data.get(f"{key}_map") def _is_required(self, data, key): @@ -679,23 +668,25 @@ def _validate_time_unit(self, data): if time: self._is_required(data, "time_unit") - def _to_internal_se(self, data): + @staticmethod + def _to_internal_se(data): input_names = ["count", "sd", "mean", "cv"] se_input = {name: data.get(name) for name in input_names} return get_se(**se_input) - def _to_internal_sd(self, data): + @staticmethod + def _to_internal_sd(data): input_names = ["count", "se", "mean", "cv"] sd_input = {name: data.get(name) for name in input_names} return get_sd(**sd_input) - def _to_internal_cv(self, data): + @staticmethod + def _to_internal_cv(data): input_names = ["count", "sd", "mean", "se"] cv_input = {name: data.get(name) for name in input_names} return get_cv(**cv_input) def _add_statistic_values(self, data, count): - se = data.get("se") sd = data.get("sd") cv = data.get("cv") @@ -714,8 +705,6 @@ def _add_statistic_values(self, data, count): return data - - @staticmethod def ex_mapping(): return { @@ -757,6 +746,7 @@ def retransform_ex_fields(cls, data): def to_internal_value(self, data): # change keys + validate_dict(data) data = self.transform_ex_fields(data) return super().to_internal_value(data) @@ -794,20 +784,27 @@ def is_valid(self, raise_exception=False): # data={something} proceed as usual return super().is_valid(raise_exception) + class ReadSerializer(serializers.HyperlinkedModelSerializer): def to_representation(self, instance): rep = super().to_representation(instance) for key,value in rep.items(): - if isinstance(value,float): - rep[key] = round(value,2) + if isinstance(value, float): + rep[key] = round(value, 2) return rep + class PkSerializer(serializers.Serializer): pk = serializers.IntegerField() class Meta: - fields = ["pk",] - + fields = ["pk", ] +def validate_dict(dic): + if not isinstance(dic, dict): + raise serializers.ValidationError( + {"error": "data must be a dictionary", + "detail": dic} + ) diff --git a/backend/pkdb_app/settings.py b/backend/pkdb_app/settings.py index 61908858..a5a1f5b9 100755 --- a/backend/pkdb_app/settings.py +++ b/backend/pkdb_app/settings.py @@ -1,9 +1,11 @@ """ Shared django settings. """ +import logging import os from os.path import join + import dj_database_url from distutils.util import strtobool @@ -239,6 +241,7 @@ REST_FRAMEWORK = { #'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', "DEFAULT_PAGINATION_CLASS": "pkdb_app.pagination.CustomPagination", + 'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.coreapi.AutoSchema', "PAGE_SIZE": int(os.getenv("DJANGO_PAGINATION_LIMIT", 20)), 'PAGINATE_BY': 10, # Default to 10 'PAGINATE_BY_PARAM': 'page_size', # Allow client to override, using `?page_size=xxx`. @@ -282,6 +285,8 @@ } DJANGO_CONFIGURATION = os.environ['PKDB_DJANGO_CONFIGURATION'] +logging.info(f"DJANGO_CONFIGURATION: {DJANGO_CONFIGURATION}") +print(f"DJANGO_CONFIGURATION: {DJANGO_CONFIGURATION}") # ------------------------------ # LOCAL # ------------------------------ diff --git a/backend/pkdb_app/statistics.py b/backend/pkdb_app/statistics.py index abbf8c3a..7411c2f1 100644 --- a/backend/pkdb_app/statistics.py +++ b/backend/pkdb_app/statistics.py @@ -48,7 +48,6 @@ class StatisticsSerializer(serializers.BaseSerializer): """ Serializer for database statistics. """ def to_representation(self, instance): - from django.db.models import Count return { key: getattr(instance, key) for key in [ @@ -65,42 +64,3 @@ def to_representation(self, instance): } -class StatisticsData(object): - """ More complex statistics data for plots and overviews. """ - - def __init__(self, substance): - self.version = __version__ - self.studies = Study.objects.filter(substances__name__contains=substance) - self.study_count = self.studies.count() - self.reference_count = self.studies.values_list("reference").count() - self.interventions = Intervention.objects.filter(substance__name=substance).filter(normed=True) - self.intervention_count = self.interventions.count() - self.outputs = Output.objects.filter(substance__name=substance).filter(normed=True) - self.output_count = self.outputs.count() - self.timecourses = Timecourse.objects.filter(substance__name=substance).filter(normed=True) - self.timecourse_count = self.timecourses.count() - - outputs_with_substance = Output.objects.filter(raw___interventions__in=self.interventions) - self.individual_count = Individual.objects.filter(output__in=outputs_with_substance).count() - self.group_count = Group.objects.filter(output__in=outputs_with_substance).count() - - -class StatisticsDataViewSet(viewsets.ViewSet): - """ - Get database statistics including version. - """ - def list(self, request): - # substances = ["caffeine", "codeine"] - data = {} - substances = Intervention.objects.values_list("substance__name", flat=True).distinct() - substances = [x for x in substances if x is not None] - - for substance in substances: - - instance = StatisticsData(substance=substance) - serializer = StatisticsSerializer(instance) - - data[substance] = serializer.data - data = pd.DataFrame(data).T.to_dict("list") - data["labels"] = substances - return Response(data) diff --git a/backend/pkdb_app/studies/models.py b/backend/pkdb_app/studies/models.py index 0bbe197a..3b5718dd 100644 --- a/backend/pkdb_app/studies/models.py +++ b/backend/pkdb_app/studies/models.py @@ -273,3 +273,5 @@ def output_calculated_count(self): if self.outputset: return self.outputset.outputs.filter(normed=True, calculated=True).count() return 0 + + diff --git a/backend/pkdb_app/studies/serializers.py b/backend/pkdb_app/studies/serializers.py index 6717f69c..72b87914 100644 --- a/backend/pkdb_app/studies/serializers.py +++ b/backend/pkdb_app/studies/serializers.py @@ -2,6 +2,8 @@ Studies serializers. """ from elasticsearch_dsl import AttrDict + +from pkdb_app import utils from pkdb_app.outputs.models import OutputSet from pkdb_app.outputs.serializers import OutputSetSerializer, OutputSetElasticSmallSerializer from pkdb_app.users.permissions import get_study_file_permission @@ -87,7 +89,7 @@ def to_internal_value(self, data): class CuratorRatingSerializer(serializers.ModelSerializer): rating = serializers.FloatField(min_value=0, max_value=5) - user = serializers.SlugRelatedField( + user = utils.SlugRelatedField( queryset=User.objects.all(), slug_field="username" ) @@ -119,13 +121,13 @@ class StudySerializer(SidSerializer): """ - reference = serializers.SlugRelatedField(slug_field="sid", + reference = utils.SlugRelatedField(slug_field="sid", queryset=Reference.objects.all(), required=True, allow_null=False ) groupset = GroupSetSerializer(read_only=False, required=False, allow_null=True) curators = CuratorRatingSerializer(many=True, required=False, allow_null=True) - collaborators = serializers.SlugRelatedField( + collaborators = utils.SlugRelatedField( queryset=User.objects.all(), slug_field="username", many=True, @@ -133,7 +135,7 @@ class StudySerializer(SidSerializer): allow_null=True, ) - creator = serializers.SlugRelatedField( + creator = utils.SlugRelatedField( queryset=User.objects.all(), slug_field="username", required=False, diff --git a/backend/pkdb_app/studies/views.py b/backend/pkdb_app/studies/views.py index 1949fbd6..d43839c9 100644 --- a/backend/pkdb_app/studies/views.py +++ b/backend/pkdb_app/studies/views.py @@ -28,6 +28,7 @@ from rest_framework.parsers import MultiPartParser, FormParser, JSONParser from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Q as DQ from elasticsearch_dsl.query import Q @@ -61,14 +62,36 @@ class StudyViewSet(viewsets.ModelViewSet): lookup_field = "sid" permission_classes = (StudyPermission,) + def get_queryset(self): + queryset = super().get_queryset() + group = user_group(self.request.user) + if group in ["admin","reviewer"]: + return queryset + + elif group == "basic": + + return queryset.filter(DQ(access=PUBLIC) | + DQ(creator=self.request.user) | + DQ(collaborators=self.request.user) | + DQ(curators=self.request.user)).distinct() + + elif group == "anonymous": + return queryset.filter(access=PUBLIC) + + @staticmethod def group_validation(request): if "groupset" in request.data and request.data["groupset"]: groupset = request.data["groupset"] if "groups" in groupset: groups = groupset.get("groups", []) + if not isinstance(groups,list): + raise ValidationError( + {"groups": f"groups must be a list and not a {type(groups)}","detail":groups}) + parents_name = set() groups_name = set() + for group in groups: parent_name = group.get("parent") if parent_name: @@ -280,6 +303,7 @@ def get_queryset(self): return qs + class ElasticReferenceViewSet(DocumentViewSet): """Read/query/search references. """ document_uid_field = "sid__raw" diff --git a/backend/pkdb_app/subjects/managers.py b/backend/pkdb_app/subjects/managers.py index 718141f0..277f65fc 100644 --- a/backend/pkdb_app/subjects/managers.py +++ b/backend/pkdb_app/subjects/managers.py @@ -13,15 +13,10 @@ def create(self, *args, **kwargs): descriptions = kwargs.pop("descriptions", []) comments = kwargs.pop("comments", []) - #Comment = apps.get_model('comments', 'Comment') - #Description = apps.get_model('comments', 'Description') - groupset = super().create(*args, **kwargs) - create_multiple(groupset, descriptions, "descriptions") create_multiple(groupset, comments, "comments") - #create_multiple_bulk(groupset, "groupset", descriptions, Description) - #create_multiple_bulk(groupset, "groupset", comments, Comment) + study_group_exs = [] @@ -34,6 +29,7 @@ def create(self, *args, **kwargs): # create single group_ex group_ex["study"] = study group_ex["group_exs"] = study_group_exs + study_group_ex = groupset.group_exs.create(**group_ex) study_group_exs.append(study_group_ex) diff --git a/backend/pkdb_app/subjects/models.py b/backend/pkdb_app/subjects/models.py index e6ce631b..d4cab0ec 100644 --- a/backend/pkdb_app/subjects/models.py +++ b/backend/pkdb_app/subjects/models.py @@ -13,7 +13,7 @@ from pkdb_app.categorials.behaviours import Normalizable, ExMeasurementTypeable -from ..utils import CHAR_MAX_LENGTH +from ..utils import CHAR_MAX_LENGTH, CHAR_MAX_LENGTH_LONG from .managers import ( GroupExManager, GroupSetManager, @@ -107,9 +107,9 @@ class GroupEx(Externable, AbstractGroup): parent_ex = models.ForeignKey("GroupEX", null=True, on_delete=models.CASCADE) name = models.CharField(max_length=CHAR_MAX_LENGTH) - name_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - count = models.IntegerField() - count_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + name_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) + count = models.IntegerField(null=True) + count_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) objects = GroupExManager() @@ -210,13 +210,12 @@ class IndividualEx(Externable, AbstractIndividual): """ source = models.ForeignKey( DataFile, related_name="s_individual_exs", null=True, on_delete=models.CASCADE) - format = models.CharField(max_length=CHAR_MAX_LENGTH, null=True, blank=True) figure = models.ForeignKey(DataFile, related_name="f_individual_exs", null=True, on_delete=models.CASCADE) individualset = models.ForeignKey(IndividualSet, on_delete=models.CASCADE, related_name="individual_exs") group = models.ForeignKey(Group, on_delete=models.CASCADE, related_name="individual_exs", null=True) - group_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + group_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) name = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) - name_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + name_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) objects = IndividualExManager() @@ -323,7 +322,7 @@ class CharacteristicaEx( This stores the raw information. Derived values can be calculated. """ - count_map = models.CharField(max_length=CHAR_MAX_LENGTH, null=True) + count_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True) group_ex = models.ForeignKey( GroupEx, related_name="characteristica_ex", null=True, on_delete=models.CASCADE diff --git a/backend/pkdb_app/subjects/serializers.py b/backend/pkdb_app/subjects/serializers.py index f74bc6c8..b64c2694 100644 --- a/backend/pkdb_app/subjects/serializers.py +++ b/backend/pkdb_app/subjects/serializers.py @@ -1,4 +1,4 @@ -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.db.models import Q from pkdb_app.categorials.behaviours import map_field, VALUE_FIELDS_NO_UNIT, \ MEASUREMENTTYPE_FIELDS, EX_MEASUREMENTTYPE_FIELDS @@ -10,7 +10,7 @@ CommentElasticSerializer from ..studies.models import Study from operator import itemgetter -from ..utils import list_of_pk +from ..utils import list_of_pk, _validate_requried_key from .models import ( Group, @@ -23,7 +23,7 @@ CharacteristicaEx, GroupEx, ) -from ..serializers import WrongKeyValidationSerializer, ExSerializer, ReadSerializer +from ..serializers import WrongKeyValidationSerializer, ExSerializer, ReadSerializer, validate_dict CHARACTERISTISTA_FIELDS = ["count"] CHARACTERISTISTA_MAP_FIELDS = map_field(CHARACTERISTISTA_FIELDS) @@ -33,7 +33,7 @@ GROUP_FIELDS = ["name", "count"] GROUP_MAP_FIELDS = ["name_map", "count_map"] -EXTERN_FILE_FIELDS = ["source", "format", "subset_map","groupby", "figure"] +EXTERN_FILE_FIELDS = ["source", "subset_map","groupby", "figure", "source_map", "figure_map"] # ---------------------------------- # DataFile @@ -122,6 +122,8 @@ def to_internal_value(self, data): data = self.retransform_map_fields(data) data = self.retransform_ex_fields(data) self.validate_wrong_keys(data) + _validate_requried_key(data,"count") + for characteristica_single in data.get("characteristica",[]): disabled = ["value"] @@ -241,6 +243,13 @@ def group_to_internal_value(group, study_sid): group = Group.objects.get( Q(ex__groupset__study__sid=study_sid) & Q(name=group) ).pk + except (ObjectDoesNotExist, MultipleObjectsReturned) as err: + if err == ObjectDoesNotExist: + msg = f'group: {group} in study: {study_sid} does not exist' + else: + msg = f'group: {group} in study: {study_sid} has been defined multiple times.' + + raise serializers.ValidationError(msg) except ObjectDoesNotExist: msg = f"group: {group} in study: {study_sid} does not exist" raise serializers.ValidationError(msg) diff --git a/backend/pkdb_app/substances/models.py b/backend/pkdb_app/substances/models.py index 3208c9d6..60e5ee15 100644 --- a/backend/pkdb_app/substances/models.py +++ b/backend/pkdb_app/substances/models.py @@ -1,17 +1,12 @@ """ Describe Substances """ -from django.apps import apps from django.db import models from pkdb_app.categorials.models import Annotation from pkdb_app.users.models import User from ..utils import CHAR_MAX_LENGTH from ..behaviours import Sidable -# ------------------------------------------------- -# Substance -# ------------------------------------------------- - class Substance(Sidable, models.Model): """ Substances. @@ -43,9 +38,7 @@ class Substance(Sidable, models.Model): annotations = models.ManyToManyField(Annotation) - # validation rule: check that all labels are in derived and not more(split on `+/()`) - def __str__(self): return self.name @@ -77,5 +70,3 @@ def creator_username(self): class SubstanceSynonym(models.Model): name = models.CharField(max_length=CHAR_MAX_LENGTH, unique=True) substance = models.ForeignKey(Substance, on_delete=models.CASCADE, related_name="synonyms") - - diff --git a/backend/pkdb_app/substances/serializers.py b/backend/pkdb_app/substances/serializers.py index 20b559bf..e9c07b77 100644 --- a/backend/pkdb_app/substances/serializers.py +++ b/backend/pkdb_app/substances/serializers.py @@ -1,6 +1,7 @@ """ Serializers for substances. """ +from pkdb_app import utils from pkdb_app.categorials.serializers import AnnotationSerializer from rest_framework import serializers @@ -30,7 +31,7 @@ def to_representation(self, instance): class SubstanceSerializer(WrongKeyValidationSerializer): """ Substance. """ - parents = serializers.SlugRelatedField(many=True, slug_field="name",queryset=Substance.objects.order_by('name'), required=False, allow_null=True) + parents = utils.SlugRelatedField(many=True, slug_field="name",queryset=Substance.objects.order_by('name'), required=False, allow_null=True) synonyms = SynonymSerializer(many=True, read_only=False, required=False, allow_null=True) annotations = AnnotationSerializer(many=True, read_only=False, required=False, allow_null=True) diff --git a/backend/pkdb_app/tests/test_study/study.json b/backend/pkdb_app/tests/test_study/study.json index cf93b467..2c2c39b3 100644 --- a/backend/pkdb_app/tests/test_study/study.json +++ b/backend/pkdb_app/tests/test_study/study.json @@ -145,14 +145,12 @@ { "name": "col==subject", "group": "col==group", - "source": "test_study_TabA.csv", - "format": "TSV" + "source": "test_study_TabA.csv" }, { "name": "col==subject", "group": "col==fake_group", "source": "test_study_Tab3.csv", - "format": "TSV", "figure": "test_study_Tab3.png", "characteristica": [ { @@ -244,7 +242,6 @@ { "source": "test_study_Tab1.csv", "subset": "intgroup==NO_OC || intgroup==A || intgroup==B", - "format": "TSV", "figure": "test_study_Tab1.png", "individual": "col==subject", "interventions": "Dcaf,DB || Dcaf || Dcaf", @@ -256,7 +253,6 @@ }, { "source": "test_study_Tab2.csv", - "format": "TSV", "subset": "intgroup==NO_OC", "figure": "test_study_Tab2.png", "group": "col==group", @@ -272,7 +268,6 @@ }, { "source": "test_study_Tab2.csv", - "format": "TSV", "subset": "intgroup==A", "figure": "test_study_Tab2.png", "group": "col==group", @@ -288,7 +283,6 @@ }, { "source": "test_study_Tab2.csv", - "format": "TSV", "subset": "intgroup==B", "figure": "test_study_Tab2.png", "group": "col==group", @@ -308,7 +302,6 @@ "groupby": "subject", "interventions": "col==interventions", "source": "test_study_Fig3.csv", - "format": "TSV", "figure": "test_study_Fig3.png", "substance": "caffeine || paraxanthine || paraxanthine/caffeine", "tissue": "plasma", @@ -327,7 +320,7 @@ "Dcaf" ], "source": "test_study_Fig3.csv", - "format": "TSV", + "figure": "test_study_Fig3.png", "substance": "caffeine || paraxanthine || paraxanthine/caffeine", "tissue": "plasma", diff --git a/backend/pkdb_app/urls.py b/backend/pkdb_app/urls.py index 9eaa60b8..96db21a1 100755 --- a/backend/pkdb_app/urls.py +++ b/backend/pkdb_app/urls.py @@ -4,7 +4,8 @@ from django.urls import path, include from django.conf.urls import url from django.contrib import admin -from pkdb_app.categorials.views import MeasurementTypeViewSet, MeasurementTypeElasticViewSet +from pkdb_app.categorials.views import MeasurementTypeViewSet, MeasurementTypeElasticViewSet, TissueViewSet, \ + ApplicationViewSet, FormViewSet, RouteViewSet from pkdb_app.outputs.views import ElasticTimecourseViewSet, ElasticOutputViewSet, OutputOptionViewSet, \ TimecourseOptionViewSet from pkdb_app.substances.views import SubstanceViewSet, ElasticSubstanceViewSet, SubstanceStatisticsViewSet @@ -12,6 +13,7 @@ from rest_framework.routers import DefaultRouter from rest_framework_swagger.views import get_swagger_view +from rest_framework.schemas import get_schema_view from .comments.views import ElasticCommentViewSet, ElasticDescriptionViewSet @@ -28,7 +30,7 @@ StudyViewSet, ElasticReferenceViewSet, ElasticStudyViewSet, update_index_study) -from .statistics import StatisticsViewSet, StatisticsDataViewSet, study_pks_view +from .statistics import StatisticsViewSet, study_pks_view router = DefaultRouter() @@ -45,6 +47,11 @@ router.register("measurement_types", MeasurementTypeViewSet, base_name="measurement_types") router.register("measurement_types_elastic", MeasurementTypeElasticViewSet, base_name="measurement_types_elastic") +router.register("tissues", TissueViewSet, base_name="tissues") +router.register("applications", ApplicationViewSet, base_name="applications") +router.register("forms", FormViewSet, base_name="forms") +router.register("routes", RouteViewSet, base_name="routes") + router.register("users", UserViewSet, base_name="users") router.register("users", UserCreateViewSet) router.register("user_groups", UserGroupViewSet, base_name="user_groups") @@ -59,7 +66,6 @@ router.register("studies_elastic", ElasticStudyViewSet, base_name="studies_elastic") router.register("statistics", StatisticsViewSet, base_name="statistics") -router.register("statistics_data", StatisticsDataViewSet, base_name="statistics_data") ############################################################################################### @@ -88,6 +94,7 @@ ) schema_view = get_swagger_view(title="PKDB API") +#schema_view = get_schema_view(title="PKDB API") urlpatterns = [ diff --git a/backend/pkdb_app/utils.py b/backend/pkdb_app/utils.py index 4267d591..07e32b13 100644 --- a/backend/pkdb_app/utils.py +++ b/backend/pkdb_app/utils.py @@ -4,10 +4,18 @@ import os import copy from rest_framework import serializers +from django.utils.translation import gettext_lazy as _ CHAR_MAX_LENGTH = 200 CHAR_MAX_LENGTH_LONG = CHAR_MAX_LENGTH * 3 + +class SlugRelatedField(serializers.SlugRelatedField): + default_error_messages = { + 'does_not_exist': _('Object with {slug_name}=<{value}> does not exist.'), + 'invalid': _('Invalid value.'), + } + def list_duplicates(seq): seen = set() seen_add = seen.add diff --git a/backend/requirements.txt b/backend/requirements.txt index 5ca0d03f..8da3542e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,7 +2,7 @@ pip>=19.2.1 # core pytz==2018.4 #old -Django==2.2.3 +Django==2.2.5 gunicorn>=19.9.0 newrelic>=3.2.2.94 @@ -31,10 +31,11 @@ coreapi>=2.3.3 # --------------------------------- # django-elasticsearch-dsl only supports elasticsearch 6 (https://github.com/sabricot/django-elasticsearch-dsl/issues/170) -elasticsearch-dsl<7.0 -django-elasticsearch-dsl>=6.4.2 +#elasticsearch-dsl<7.0 +elasticsearch-dsl==6.4.0 +django-elasticsearch-dsl==6.4.2 # see https://github.com/barseghyanartur/django-elasticsearch-dsl-drf/pull/141 -# django-elasticsearch-dsl-drf>=0.18 +#django-elasticsearch-dsl-drf==0.18 -e git+https://github.com/barseghyanartur/django-elasticsearch-dsl-drf.git#egg=django-elasticsearch-dsl-drf diff --git a/docker-compose-develop.yml b/docker-compose-develop.yml index 2b3f89d3..a21f213c 100644 --- a/docker-compose-develop.yml +++ b/docker-compose-develop.yml @@ -70,7 +70,7 @@ services: context: ./frontend dockerfile: Dockerfile-develop ports: - - "8080:8080" + - "8081:8080" volumes: - ./frontend:/app - node_modules:/app/node_modules/