Merge pull request #607 from matthiaskoenig/develop

Merge develop

.gitignore

@@ -12,10 +12,17 @@ __pycache__/
 *.ods#
 *.xlsx#
 
+# ----------------------------
+# Database dump
+# ----------------------------
+pkdb.dump
+
 # ----------------------------
 # Environments
 # ----------------------------
 .env
+.env.alpha
+.env.develop
 .env.production
 frontend/.env.production
 
@@ -53,21 +60,6 @@ MANIFEST
 .static_storage/
 media/
 
-# ----------------------------
-# django migrations
-# ----------------------------
-*/migrations/
-.mypy_cache/
-backend/pkdb_app/studies/migrations/00*.py
-backend/pkdb_app/users/migrations/00*.py
-backend/pkdb_app/subjects/migrations/00*.py
-backend/pkdb_app/interventions/migrations/00*.py
-backend/pkdb_app/comments/migrations/00*.py
-backend/pkdb_app/outputs/migrations/00*.py
-backend/pkdb_app/depricated/categorials/migrations/00*.py
-backend/pkdb_app/depricated/substances/migrations/00*.py
-backend/pkdb_app/info_nodes/migrations/00*.py
-
 
 # Installer logs
 pip-log.txt

TODO.md

@@ -0,0 +1,14 @@
+# TODO
+- [ ] show info nodes characteristica details
+- [ ] show detail views for study, group, individual, reference (fix 404 on detail buttons), remove reference button
+- [ ] Fix API url
+- [ ] Fix Layout of main component (refactor in smaller components)
+- [ ] Better REST documentation
+- [ ] better name for zip file, e.g. pkdb_data_2020-08-23.zip
+- [ ] fix data issues (missing titles), descriptions of info nodes
+
+---
+- [ ] cache info nodes in store
+- [ ] implement additional validation rules
+- [ ] filter by access or license
+- [ ] make scatters available

backend/.gitignore

@@ -0,0 +1,19 @@
+__pycache__/
+*.py[cod]
+*$py.class
+# ----------------------------
+# django migrations
+# ----------------------------
+*/migrations/
+.mypy_cache/
+pkdb_app/studies/migrations/00*.py
+pkdb_app/users/migrations/00*.py
+pkdb_app/subjects/migrations/00*.py
+pkdb_app/interventions/migrations/00*.py
+pkdb_app/comments/migrations/00*.py
+pkdb_app/outputs/migrations/00*.py
+pkdb_app/depricated/categorials/migrations/00*.py
+pkdb_app/depricated/substances/migrations/00*.py
+pkdb_app/info_nodes/migrations/00*.py
+pkdb_app/data/migrations/00*.py
+

backend/README.md

@@ -1,3 +1,43 @@
 # PKDB Backend (`django`)
+
+- [ ] documentation page with queries and searches
+
+## PKData Query
+The event cycle of PKData is:
+1. Query studies, interventions, groups, individuals, and outputs by adding
+ the respective word as a prefix following two underscores to the url filter 
+ (e.g. ...api/v1/pkdata/?studies__sid=PKDB00008 is equivalent to ...api/v1/studies/?sid=PKDB00008).
+ The search/filter is performed on the indexed database. For more details on how to construct the query by patterns in the 
+ url check "https://django-elasticsearch-dsl-drf.readthedocs.io/en/latest/".
 
-© 2017-2019 Jan Grzegorzewski & Matthias König.
+2. All tables are updated to get rid of redundant entries. This results in a concise set of entries
+in all tables (e.g. a filter on the study table for a specific sid reduces the entries of the other tables 
+only to interventions, groups, individuals, and outputs which are part of the study).
+
+3. paginated studies, interventions, groups, individuals, and outputs are returned. Getting the next page for one of the tables 
+works equivalently to the filters (e.g. getting the second studies page while searching for the interventions containing caffeine.  ...api/v1/pkdata/?interventions__substance=caffeine&studies__page=2).
+
+
+## PKDData
+documentation
+
+### Queries
+
+Query for single study:
+``` 
+http://localhost:8000/api/v1/pkdata/?studies__sid=PKDB00008 
+```
+Query for multiple studies based on sids:
+```
+http://localhost:8000/api/v1/pkdata/?studies__sid__in=PKDB00008__PKDB00001
+```
+Query for interventions substance:
+```
+http://localhost:8000/api/v1/pkdata/?interventions__substance=codeine
+```
+Query for interventions and outputs simultaneously:
+```
+http://localhost:8000/api/v1/pkdata/?interventions__substance=codeine&outputs__measurement_type=clearance
+```
+
+© 2017-2020 Jan Grzegorzewski & Matthias König.

backend/pkdb_app/_version.py

@@ -1,4 +1,4 @@
 """
 Definition of version string.
 """
-__version__ = "0.8.0"
+__version__ = "0.9.1"

backend/pkdb_app/behaviours.py

@@ -19,13 +19,13 @@ class Meta:
 class Externable(models.Model):
     # format = models.CharField(max_length=CHAR_MAX_LENGTH, null=True)
     subset_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
-    groupby = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
     source_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
-    figure_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
+    image_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
 
     class Meta:
         abstract = True
 
+
 class Accessible(models.Model):
     class Meta:
         abstract = True
@@ -98,19 +98,6 @@ class Meta:
         abstract = True
 
 
-class ExMeasurementTypeable(ValueableNotBlank, ValueableMapNotBlank):
-    measurement_type = models.CharField(max_length=CHAR_MAX_LENGTH, null=True)
-    measurement_type_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
-
-    choice = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
-    choice_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
-    substance = models.CharField(max_length=CHAR_MAX_LENGTH, null=True)
-    substance_map = models.CharField(max_length=CHAR_MAX_LENGTH_LONG, null=True)
-
-    class Meta:
-        abstract = True
-
-
 class MeasurementTypeable(ValueableNotBlank):
 
     measurement_type = models.ForeignKey('info_nodes.MeasurementType', on_delete=models.PROTECT)

backend/pkdb_app/comments/models.py

@@ -1,11 +1,12 @@
 from django.db import models
 
 # Create your models here.
+from pkdb_app.data.models import Data, DataSet, Dimension, SubSet
 from pkdb_app.interventions.models import (
     InterventionEx,
     InterventionSet,
 )
-from pkdb_app.outputs.models import OutputEx, OutputSet, TimecourseEx
+from pkdb_app.outputs.models import OutputEx, OutputSet
 from pkdb_app.studies.models import Study
 from pkdb_app.subjects.models import (
     IndividualEx,
@@ -23,32 +24,48 @@ class Comment(models.Model):
         User, related_name="comments", null=True, on_delete=models.CASCADE
     )
     date_time = models.DateTimeField(auto_now_add=True, blank=True)
-    ####
 
     individual_ex = models.ForeignKey(
         IndividualEx, related_name="comments", null=True, on_delete=models.CASCADE
     )
+
     individualset = models.ForeignKey(
         IndividualSet, related_name="comments", null=True, on_delete=models.CASCADE
     )
     group_ex = models.ForeignKey(
         GroupEx, related_name="comments", null=True, on_delete=models.CASCADE
     )
+
     groupset = models.ForeignKey(
         GroupSet, related_name="comments", null=True, on_delete=models.CASCADE
     )
+
     characteristica_ex = models.ForeignKey(
         CharacteristicaEx, related_name="comments", null=True, on_delete=models.CASCADE
     )
 
     output_ex = models.ForeignKey(
         OutputEx, related_name="comments", null=True, on_delete=models.CASCADE
     )
+
     outputset = models.ForeignKey(
         OutputSet, related_name="comments", null=True, on_delete=models.CASCADE
     )
-    timecourse_ex = models.ForeignKey(
-        TimecourseEx, related_name="comments", null=True, on_delete=models.CASCADE
+
+    datasets = models.ForeignKey(
+        DataSet, related_name="comments", null=True, on_delete=models.CASCADE
+    )
+
+    data = models.ForeignKey(
+        Data, related_name="comments", null=True, on_delete=models.CASCADE
+    )
+
+    subsets = models.ForeignKey(
+        SubSet, related_name="comments", null=True, on_delete=models.CASCADE
+    )
+
+    dimensions = models.ForeignKey(
+        Dimension, related_name="comments", null=True, on_delete=models.CASCADE
     )
 
     intervention_ex = models.ForeignKey(
@@ -84,6 +101,23 @@ class Description(models.Model):
     outputset = models.ForeignKey(
         OutputSet, related_name="descriptions", null=True, on_delete=models.CASCADE
     )
+
+    datasets = models.ForeignKey(
+        DataSet, related_name="descriptions", null=True, on_delete=models.CASCADE
+    )
+
+    data = models.ForeignKey(
+        Data, related_name="descriptions", null=True, on_delete=models.CASCADE
+    )
+
+    subsets = models.ForeignKey(
+        SubSet, related_name="descriptions", null=True, on_delete=models.CASCADE
+    )
+
+
+    dimensions = models.ForeignKey(
+        Dimension, related_name="descriptions", null=True, on_delete=models.CASCADE
+    )
     individualset = models.ForeignKey(
         IndividualSet, related_name="descriptions", null=True, on_delete=models.CASCADE
     )
@@ -110,10 +144,6 @@ class Description(models.Model):
         OutputEx, related_name="descriptions", null=True, on_delete=models.CASCADE
     )
 
-    timecourse_ex = models.ForeignKey(
-        TimecourseEx, related_name="descriptions", null=True, on_delete=models.CASCADE
-    )
-
     intervention_ex = models.ForeignKey(
         InterventionEx, related_name="descriptions", null=True, on_delete=models.CASCADE
     )

backend/pkdb_app/data/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class FiguresConfig(AppConfig):
+    name = "data"

backend/pkdb_app/data/documents.py

@@ -0,0 +1,123 @@
+from django_elasticsearch_dsl import Document, fields, ObjectField
+from django_elasticsearch_dsl.registries import registry
+from pkdb_app.data.models import Dimension, SubSet
+
+from ..documents import string_field, elastic_settings, info_node, study_field
+
+
+# ------------------------------------
+# Elastic Output Document
+# ------------------------------------
+
+
+@registry.register_document
+class DataAnalysisDocument(Document):
+    study_sid = string_field('study_sid')
+    study_name = string_field('study_name')
+
+    data_pk = fields.IntegerField('data_pk')
+    data_name = string_field('data_name')
+    data_type = string_field('data_type')
+
+    subset_pk = fields.IntegerField('subset_pk')
+    subset_name = string_field('subset_name')
+
+    data_point_pk = fields.IntegerField('data_point_pk')
+    output_pk = fields.IntegerField('output_pk')
+    dimension = fields.IntegerField('dimension')
+
+    # for permissions
+    access = string_field('access')
+    allowed_users = fields.ObjectField(
+        attr="allowed_users",
+        properties={
+            'username': string_field("username")
+        },
+        multi=True
+    )
+
+    class Django:
+        model = Dimension
+        # Ignore auto updating of Elasticsearch when a model is saved/deleted
+        ignore_signals = True
+        # Don't perform an index refresh after every update
+        auto_refresh = False
+
+    class Index:
+        name = 'data_analysis'
+        settings = elastic_settings
+        settings['number_of_shards'] = 5
+        settings['number_of_replicas'] = 1
+        settings['max_result_window'] = 100000
+
+
+output_field = dict(
+    pk = fields.IntegerField(),
+    group = ObjectField(properties={
+        'pk': fields.IntegerField(),
+        'name': string_field('name'),
+        'count': fields.IntegerField(),
+    }),
+    individual=ObjectField(properties={
+        'pk': fields.IntegerField(),
+        'name': string_field('name')}),
+    interventions=ObjectField(properties={
+        'pk': fields.IntegerField(),
+        'name': string_field('name') }, multi=True),
+
+    ex=ObjectField(properties={'pk': string_field('pk')}),
+    normed=fields.BooleanField(),
+    value=fields.FloatField('null_value'),
+    mean=fields.FloatField('null_mean'),
+    median=fields.FloatField('null_median'),
+    min=fields.FloatField('null_min'),
+    max=fields.FloatField('null_max'),
+    se=fields.FloatField('null_se'),
+    sd=fields.FloatField('null_sd'),
+    cv=fields.FloatField('null_cv'),
+    unit=string_field('unit'),
+    time_unit=string_field('time_unit'),
+    time=fields.FloatField('null_time'),
+    tissue=info_node('i_tissue'),
+    method=info_node('i_method'),
+    measurement_type=info_node('i_measurement_type'),
+    substance=info_node('i_substance'),
+    choice=info_node('i_choice'),
+    label=string_field('label'))
+
+@registry.register_document
+class SubSetDocument(Document):
+    pk = fields.IntegerField()
+    array = fields.ObjectField(
+        attr="data_points",
+        properties={
+            'point': fields.ObjectField(attr="outputs", properties=output_field)},
+        multi=True)
+    data_type = string_field('data_type')
+    name = string_field('name')
+    study = study_field
+    study_sid = string_field('study_sid')
+    study_name = string_field('study_name')
+    # for permissions
+    access = string_field('access')
+    allowed_users = fields.ObjectField(
+        attr="allowed_users",
+        properties={
+            'username': string_field("username")
+        },
+        multi=True
+    )
+
+    class Django:
+        model = SubSet
+        # Ignore auto updating of Elasticsearch when a model is saved/deleted
+        ignore_signals = True
+        # Don't perform an index refresh after every update
+        auto_refresh = False
+
+    class Index:
+        name = 'subset'
+        settings = elastic_settings
+        settings['number_of_shards'] = 5
+        settings['number_of_replicas'] = 1
+        settings['max_result_window'] = 100000