Merge pull request #702 from matthiaskoenig/develop

Changes for v0.9.5

backend/pkdb_app/_version.py

@@ -1,4 +1,4 @@
 """
 Definition of version string.
 """
-__version__ = "0.9.4"
+__version__ = "0.9.5"

backend/pkdb_app/data/serializers.py

@@ -70,9 +70,14 @@ def create(self, validated_data):
         # subset_instance.save()
         return subset_instance
 
+    def validate_shared(self, shared):
+        if isinstance(shared,str):
+            raise serializers.ValidationError({"shared": "Shared field has to be a list not a string.", "detail": shared})
+
+
     def _validate_time(self, time):
         if any(np.isnan(np.array(time))):
-            raise serializers.ValidationError({"time": "no time points are allowed to be nan", "detail": time})
+            raise serializers.ValidationError({"time": "No time points are allowed to be nan", "detail": time})
 
     def calculate_pks_from_timecourses(self, subset):
         # calculate pharmacokinetics outputs

backend/pkdb_app/error_measures.py

@@ -6,16 +6,19 @@
     cv is coefficient of variation. sd/mean
 """
 import numpy as np
-import warnings
+
+
+def _is(value):
+    return value is not None and value is not np.nan
 
 
 def calculate_sd(se, count, cv, mean):
     """Calculates standard deviation from other error measurements."""
     sd = None
-    is_se = sd is not None
-    is_count = count is not None
-    is_mean = mean is not None
-    is_cv = cv is not None
+    is_se = _is(se)
+    is_count = _is(count)
+    is_mean = _is(mean)
+    is_cv = _is(cv)
 
     if is_se and is_count:
         sd = np.multiply(se, np.sqrt(count))
@@ -28,10 +31,10 @@ def calculate_se(sd, count, cv, mean):
     """Calculates SE from given fields."""
 
     se = None
-    is_sd = sd is not None
-    is_count = count is not None
-    is_mean = mean is not None
-    is_cv = cv is not None
+    is_sd = _is(sd)
+    is_count = _is(count)
+    is_mean = _is(mean)
+    is_cv = _is(cv)
 
     if is_sd and is_count:
         se = np.true_divide(sd, np.sqrt(count))
@@ -44,10 +47,10 @@ def calculate_cv(sd, count, se, mean):
     """Calculates CV from given fields"""
 
     cv = None
-    is_sd = sd is not None
-    is_count = count is not None
-    is_mean = mean is not None
-    is_se = se is not None
+    is_sd = _is(sd)
+    is_count = _is(count)
+    is_mean = _is(mean)
+    is_se = _is(se)
 
     # mean can be zero, CV not calculatable, resulting in -inf/inf
     # mean data must be cleaned before calculation

backend/pkdb_app/info_nodes/documents.py

@@ -41,16 +41,13 @@ class InfoNodeDocument(Document):
     ntype = string_field('ntype')
     dtype = string_field('dtype')
     xrefs = ObjectField(
-        properties=
-                          {
-                              "name": string_field("name"),
-                              "accession":string_field("accession"),
-                              "url":string_field("url")
-
-                          }, multi=True)
-
-
-    # measurement type
+        properties={
+            "name": string_field("name"),
+            "accession": string_field("accession"),
+            "url": string_field("url")
+        },
+        multi=True
+    )
     measurement_type = ObjectField(
         properties={
             "choices": ObjectField(
@@ -61,15 +58,12 @@ class InfoNodeDocument(Document):
             "units": basic_object("units", multi=True)
         }
     )
-    # substance
     substance = ObjectField(
         properties={
-
             "chebi": string_field('chebi'),
             "mass": string_field('mass'),
             "charge": string_field('charge'),
             "formula": string_field('formula'),
-
         })
 
     class Django:

backend/pkdb_app/info_nodes/models.py

@@ -11,7 +11,7 @@
 
 from pkdb_app.behaviours import Sidable
 from pkdb_app.info_nodes.units import ureg
-from pkdb_app.utils import CHAR_MAX_LENGTH, CHAR_MAX_LENGTH_LONG, \
+from pkdb_app.utils import CHAR_MAX_LENGTH, CHAR_MAX_LENGTH_LONG, _validate_required_key_and_value_or_nr, \
     _validate_required_key_and_value
 from rest_framework import serializers
 
@@ -153,8 +153,18 @@ class MeasurementType(AbstractInfoNode):
     )
 
     NO_UNIT = 'NO_UNIT'  # todo: remove NO_UNIT and add extra keyword or add an extra measurement_type with optional no units.
-    TIME_REQUIRED_MEASUREMENT_TYPES = ["cumulative amount", "cumulative metabolic ratio", "recovery",
-                                       "auc_end"]  # todo: remove and add extra keyword.
+    TIME_REQUIRED_MEASUREMENT_TYPES = [
+        "concentration",
+        "cumulative amount",
+        "metabolic ratio",
+        "cumulative metabolic ratio",
+        "recovery",
+        "auc_end",
+        "ptf",
+    ]  # todo: remove and add extra keyword.
+    TIME_REQUIRED_MEASUREMENT_TYPES_NR_ALLOWED = [
+
+     ]
     CAN_NEGATIVE = [
         "tmax"  # tmax can be negative due to time offsets, i.e. pre-simulation with subsequent fall after intervention
                 # this often happens in placebo simulations
@@ -315,7 +325,7 @@ def validate_choice(self, choice):
                                         self.info_node.DTypes.NumericCategorical]:
                 if not self.is_valid_choice(choice):
                     msg = f"The choice `{choice}` is not a valid choice for measurement type `{self.info_node.name}`. " \
-                          f"Allowed choices are: `{list(self.choices_list())}`."
+                          f"Allowed choices are: `{sorted(self.choices_list())}`."
                     raise ValueError({"choice": msg})
                 return self.choices.get(info_node__name=choice)
             else:
@@ -325,7 +335,7 @@ def validate_choice(self, choice):
                 raise ValueError({"choice": msg})
         elif self.choices.exists():
             msg = f"{choice}. A choice is required for `{self.info_node.name}`." \
-                  f" Allowed choices are: `{list(self.choices_list())}`."
+                  f" Allowed choices are: `{sorted(self.choices_list())}`."
             raise ValueError({"choice": msg})
 
     @property
@@ -358,7 +368,7 @@ def validate_numeric(self, data):
                                     f"for all measurement types except "
                                     f"<{self.CAN_NEGATIVE}>.", "detail": data})
 
-    def validate_complete(self, data):
+    def validate_complete(self, data, time_allowed: bool = True):
         """Complete validation."""
 
         # check unit
@@ -368,16 +378,27 @@ def validate_complete(self, data):
         choice = data.get("choice", None)
         d_choice = self.validate_choice(choice)
 
-        time_unit = data.get("time_unit", None)
-        if time_unit:
-            self.validate_time_unit(time_unit)
+        if time_allowed:
+            time_unit = data.get("time_unit", None)
+            time = data.get("time", None)
 
-        if self.time_required:
-            details = f"for measurement type `{self.info_node.name}`"
-            _validate_required_key_and_value(data, "time", details=details)
-            _validate_required_key_and_value(data, "time_unit", details=details)
+            if time_unit and time_unit != "NR":
+                self.validate_time_unit(time_unit)
 
-        return {"choice":d_choice}
+            if self.time_required:
+                details = f"for measurement type `{self.info_node.name}`"
+
+                if time != "NR":
+                    _validate_required_key_and_value(data, "time", details=details)
+                    if time_unit != "NR":
+                        _validate_required_key_and_value(data, "time_unit", details=details)
+
+            if time == "NR":
+                data["time"] = None
+            if time_unit == "NR":
+                data["time_unit"] = None
+
+        return {"choice": d_choice}
 
 
 class Choice(AbstractInfoNode):

backend/pkdb_app/info_nodes/serializers.py

@@ -4,7 +4,7 @@
 from pkdb_app.info_nodes.documents import InfoNodeDocument
 from pkdb_app.info_nodes.models import InfoNode, Synonym, Annotation, Unit, MeasurementType, Substance, Choice, Route, \
     Form, Tissue, Application, Method, CrossReference
-from pkdb_app.serializers import WrongKeyValidationSerializer, ExSerializer, SidNameLabelSerializer
+from pkdb_app.serializers import WrongKeyValidationSerializer, ExSerializer, SidNameLabelSerializer, FloatNRField
 from pkdb_app.utils import update_or_create_multiple
 from rest_framework.fields import empty
 
@@ -28,17 +28,14 @@ class MeasurementTypeableSerializer(EXMeasurementTypeableSerializer):
     )
 
     choice = serializers.CharField(allow_null=True)
-
-    def to_representation(self, instance):
-        rep = super().to_representation(instance)
-        return rep
+    time = FloatNRField(allow_null=True)
 
 
 class SynonymSerializer(WrongKeyValidationSerializer):
     pk = serializers.IntegerField(read_only=True)
     class Meta:
         model = Synonym
-        fields = ["name","pk"]
+        fields = ["name", "pk"]
 
     def to_internal_value(self, data):
         return {"name": data}
@@ -247,4 +244,13 @@ class Meta:
         fields = ["sid", "name", "label", "deprecated", "ntype", "dtype", "description", "synonyms", "parents", "annotations", "xrefs","measurement_type", "substance",  ]
 
     def get_synonyms(self, obj):
-        return [synonym["name"] for synonym in obj.synonyms]
+        return [synonym["name"] for synonym in obj.synonyms]
+
+
+class IndoNodeFlatSerializer(serializers.Serializer):
+    sid = serializers.CharField()
+    label = serializers.CharField()
+    ntype = serializers.CharField()
+
+    class Meta:
+        fields =["sid", "name", "label", "ntype", "dtype"]

backend/pkdb_app/interventions/serializers.py

@@ -2,6 +2,7 @@
 Serializers for interventions.
 """
 import itertools
+from abc import ABC
 
 from rest_framework import serializers
 
@@ -108,7 +109,7 @@ def validate(self, attrs):
             for info_node in ['substance', 'measurement_type', 'form', 'application', 'route']:
                 if info_node in attrs:
                     if attrs[info_node] is not None:
-                        attrs[info_node] = getattr(attrs[info_node],info_node)
+                        attrs[info_node] = getattr(attrs[info_node], info_node)
 
             attrs["choice"] = attrs["measurement_type"].validate_complete(data=attrs)["choice"]
 
@@ -165,7 +166,7 @@ def to_internal_value(self, data):
             interventions_from_file = self.entries_from_file(intervention)
             interventions.extend(interventions_from_file)
 
-        drop_fields =  INTERVENTION_FIELDS + INTERVENTION_MAP_FIELDS + EX_MEASUREMENTTYPE_FIELDS
+        drop_fields = INTERVENTION_FIELDS + INTERVENTION_MAP_FIELDS + EX_MEASUREMENTTYPE_FIELDS
         [data.pop(field, None) for field in drop_fields]
         # ----------------------------------
         # finished
@@ -320,7 +321,7 @@ class InterventionElasticSerializerAnalysis(serializers.Serializer):
     measurement_type_label = serializers.SerializerMethodField()
 
     choice = serializers.SerializerMethodField()
-    choice_label =serializers.SerializerMethodField()
+    choice_label = serializers.SerializerMethodField()
 
     substance = serializers.SerializerMethodField()
     substance_label = serializers.SerializerMethodField()

backend/pkdb_app/outputs/models.py

@@ -109,7 +109,7 @@ class OutputTypes(models.TextChoices):
 
     group = models.ForeignKey(Group, null=True, blank=True, on_delete=models.CASCADE)
     individual = models.ForeignKey(Individual, null=True, blank=True, on_delete=models.CASCADE)
-    interventions = models.ManyToManyField(Intervention, through="OutputIntervention", related_name="outputs")
+    interventions = models.ManyToManyField(Intervention, through="OutputIntervention", related_name="outputs", null=True, blank=True)
     subset = models.ForeignKey('data.Subset', on_delete=models.CASCADE, null=True, blank=True, related_name="pks")
 
     tissue = models.ForeignKey(Tissue, related_name="outputs", null=True, blank=True, on_delete=models.CASCADE)
@@ -130,7 +130,7 @@ def null_attr(self, attr):
 
     def is_timecourse(self):
         if self.timecourse:
-            if self.timecourse.data.data_type=="timecourse":
+            if self.timecourse.data.data_type == "timecourse":
                 return True
         return False
 

backend/pkdb_app/outputs/serializers.py

@@ -29,7 +29,7 @@
 from ..utils import list_of_pk, _validate_required_key, create_multiple, _create, create_multiple_bulk_normalized, \
     create_multiple_bulk
 
-EXTRA_FIELDS = ["tissue", "method", "label","output_type"]
+EXTRA_FIELDS = ["tissue", "method", "label", "output_type"]
 TIME_FIELDS = ["time", "time_unit"]
 OUTPUT_FIELDS = EXTRA_FIELDS + TIME_FIELDS
 
@@ -51,6 +51,7 @@
 # Outputs
 # ----------------------------------
 
+
 class OutputSerializer(MeasurementTypeableSerializer):
 
     group = serializers.PrimaryKeyRelatedField(
@@ -64,11 +65,8 @@ class OutputSerializer(MeasurementTypeableSerializer):
     )
     interventions = serializers.PrimaryKeyRelatedField(
         queryset=Intervention.objects.all(),
-        many=True,
-        read_only=False,
-        required=False,
-        allow_null=True,
-    )
+        many=True, required=True, allow_null=True)
+
     tissue = utils.SlugRelatedField(
         slug_field="name",
         queryset=InfoNode.objects.filter(ntype=InfoNode.NTypes.Tissue),
@@ -91,10 +89,12 @@ class Meta:
     def to_internal_value(self, data):
         data.pop("comments", None)
         data.pop("descriptions", None)
+        data.pop("image", None)
+        data.pop("source", None)
 
         data = self.retransform_map_fields(data)
         data = self.to_internal_related_fields(data)
-        self.validate_wrong_keys(data, additional_fields=OutputExSerializer.Meta.fields)
+        self.validate_wrong_keys(data)
         return super(serializers.ModelSerializer, self).to_internal_value(data)
 
     def validate(self, attrs):
@@ -103,24 +103,19 @@ def validate(self, attrs):
         self.validate_group_individual_output(attrs)
 
         _validate_required_key(attrs, "measurement_type")
-
         _validate_required_key(attrs, "substance")
         _validate_required_key(attrs, "tissue")
         _validate_required_key(attrs, "interventions")
         _validate_required_key(attrs, "output_type")
         self._validate_timecourse(attrs)
 
-
-
         try:
             attrs['measurement_type'] = attrs['measurement_type'].measurement_type
 
             for key in ['substance', 'tissue', 'method']:
                 if key in attrs:
                     if attrs[key] is not None:
                         attrs[key] = getattr(attrs[key], key)
-
-
             attrs["choice"] = attrs["measurement_type"].validate_complete(data=attrs)["choice"]
 
         except ValueError as err:
@@ -177,7 +172,6 @@ def to_internal_value(self, data):
         for output in temp_outputs:
             outputs_from_file = self.entries_from_file(output)
             outputs.extend(outputs_from_file)
-
         # ----------------------------------
         # finished
         # ----------------------------------