adding RankResponse model

config/settings/base.py

@@ -342,3 +342,11 @@
     "CHR_FORMAT",
     default="ucsc",
 )
+CHIPEXO_PROMOTER_SIG_FORMAT = env(
+    "CHIPEXO_PROMOTER_SIG_FORMAT",
+    default="chipexo_promoter_sig",
+)
+CALLINGCARDS_PROMOTER_SIG_FORMAT = env(
+    "CALLING_CARD_PROMOTER_SIG_FORMAT",
+    default="callingcards_promoter_sig",
+)

docs/diagram_exclude_models.txt

@@ -0,0 +1,2 @@
+User
+BaseModel

local.yml

@@ -19,6 +19,7 @@ services:
     env_file:
       - ./.envs/.local/.django
       - ./.envs/.local/.postgres
+      - ./.envs/.local/.regulatory_data
     ports:
       - '8000:8000'
     command: /start

production.yml

@@ -18,6 +18,7 @@ services:
     env_file:
       - ./.envs/.production/.django
       - ./.envs/.production/.postgres
+      - ./.envs/.production/.regulatory_data
     command: /start
 
   postgres:

yeastregulatorydb/conftest.py

@@ -185,7 +185,9 @@ def fileformat(db) -> QuerySet:
             },
             ",",
             "max_fc",
+            0.0,
             "min_pval",
+            1.0,
         ),
         "cc_promoter_sig": (
             {
@@ -204,13 +206,17 @@ def fileformat(db) -> QuerySet:
             },
             ",",
             "callingcards_enrichment",
+            0.0,
             "poisson_pval",
+            1.0,
         ),
         "kemmeren": (
             {"gene_id": "int", "M": "float", "Madj": "float", "A": "float", "pval": "float"},
             ",",
             "Madj",
+            0.0,
             "pval",
+            1.0,
         ),
         "mcisaac": (
             {
@@ -224,18 +230,47 @@ def fileformat(db) -> QuerySet:
             },
             ",",
             "log2_shrunken_timecourses",
+            0.0,
             "none",
+            1.0,
         ),
         "bed6": (
             {"chr": "str", "start": "int", "end": "int", "name": "str", "score": "float", "strand": "str"},
             "\t",
             "none",
+            0.0,
             "none",
+            1.0,
+        ),
+        "rank_response_summary": (
+            {
+                "feature": "str",
+                "expression_effect": "int",
+                "expression_pvalue": "int",
+                "binding_effect": "str",
+                "binding_pvalue": "str",
+                "responsive": "int",
+                "ran_bin": "float",
+                "random": "float",
+            },
+            ",",
+            "none",
+            0.0,
+            "none",
+            1.0,
         ),
     }
     for key, value in format_dict.items():
-        fields, separator, effect, pval = value
-        FileFormatFactory.create(fileformat=key, fields=fields, separator=separator, effect_col=effect, pval_col=pval)
+        fields, separator, effect, effect_thres, pval, pval_thres = value
+        FileFormatFactory.create(
+            fileformat=key,
+            fields=fields,
+            separator=separator,
+            effect_col=effect,
+            default_effect_threshold=effect_thres,
+            pval_col=pval,
+            default_pvalue_threshold=pval_thres,
+        )
     return FileFormat.objects.all()
 
 

yeastregulatorydb/regulatory_data/api/filters/RankResponseFilter.py

@@ -0,0 +1,37 @@
+import django_filters
+
+from ...models import RankResponse
+
+
+class RankResponseFilter(django_filters.FilterSet):
+    id = django_filters.NumberFilter()
+    pk = django_filters.NumberFilter()
+    promotersetsig_id = django_filters.NumberFilter(field_name="promotersetsig__id")
+    binding_source = django_filters.CharFilter(
+        field_name="promotersetsig__binding__source__name", lookup_expr="iexact"
+    )
+    expression_id = django_filters.NumberFilter(field_name="expression__id")
+    expression_source = django_filters.CharFilter(field_name="expression__source__name", lookup_expr="iexact")
+    regulator_locus_tag = django_filters.CharFilter(
+        field_name="expression__regulator__locus_tag", lookup_expr="iexact"
+    )
+    regulator_symbol = django_filters.CharFilter(field_name="expression__regulator__symbol", lookup_expr="iexact")
+    expression_effect_threshold = django_filters.NumberFilter()
+    expression_pvalue_threshold = django_filters.NumberFilter()
+    normalized = django_filters.BooleanFilter()
+
+    class Meta:
+        model = RankResponse
+        fields = [
+            "id",
+            "pk",
+            "promotersetsig_id",
+            "binding_source",
+            "expression_id",
+            "expression_source",
+            "regulator_locus_tag",
+            "regulator_symbol",
+            "expression_effect_threshold",
+            "expression_pvalue_threshold",
+            "normalized",
+        ]

yeastregulatorydb/regulatory_data/api/serializers/PromoterSetSigSerializer.py

@@ -1,10 +1,6 @@
-import pandas as pd
-from django.conf import settings
 from rest_framework import serializers
 
-from yeastregulatorydb.regulatory_data.utils.validate_genomic_df import validate_genomic_df
-
-from ...models import FileFormat, PromoterSetSig
+from ...models import PromoterSetSig
 from .mixins.CustomValidateMixin import CustomValidateMixin
 from .mixins.FileValidationMixin import FileValidationMixin
 

yeastregulatorydb/regulatory_data/api/serializers/RankResponseSerializer.py

@@ -0,0 +1,14 @@
+from rest_framework import serializers
+
+from ...models import RankResponse
+from .mixins.CustomValidateMixin import CustomValidateMixin
+from .mixins.FileValidationMixin import FileValidationMixin
+
+
+class RankResponseSerializer(CustomValidateMixin, FileValidationMixin, serializers.ModelSerializer):
+    uploader = serializers.ReadOnlyField(source="uploader.username")
+    modifier = serializers.CharField(source="uploader.username", required=False)
+
+    class Meta:
+        model = RankResponse
+        fields = "__all__"

yeastregulatorydb/regulatory_data/api/serializers/__init__.py

@@ -9,6 +9,7 @@
 from .GenomicFeatureSerializer import GenomicFeatureSerializer
 from .PromoterSetSerializer import PromoterSetSerializer
 from .PromoterSetSigSerializer import PromoterSetSigSerializer
+from .RankResponseSerializer import RankResponseSerializer
 from .RegulatorSerializer import RegulatorSerializer
 
 __all__ = [
@@ -23,5 +24,5 @@
     "GenomicFeatureSerializer",
     "PromoterSetSerializer",
     "PromoterSetSigSerializer",
-    "RegulatorSerializer",
+    "RankResponseSerializer" "RegulatorSerializer",
 ]

yeastregulatorydb/regulatory_data/api/views/BindingViewSet.py

@@ -1,14 +1,13 @@
+from celery import chain
+from django.core.cache import cache
 from django_filters.rest_framework import DjangoFilterBackend
 from rest_framework import viewsets
 from rest_framework.authentication import SessionAuthentication, TokenAuthentication
 from rest_framework.permissions import IsAuthenticated
 
-from yeastregulatorydb.regulatory_data.tasks.chipexo_pugh_allevents_promoter_sig import (
-    chipexo_pugh_allevents_promoter_sig,
-)
-
 from ...models.Binding import Binding
-from ..filters.BindingFilter import BindingFilter
+from ...tasks import promoter_significance_task, rank_response_tasks
+from ..filters import BindingFilter
 from ..serializers.BindingSerializer import BindingSerializer
 from .mixins.BulkUploadMixin import BulkUploadMixin
 from .mixins.UpdateModifiedMixin import UpdateModifiedMixin
@@ -28,5 +27,24 @@ class BindingViewSet(UpdateModifiedMixin, viewsets.ModelViewSet, BulkUploadMixin
 
     def perform_create(self, serializer):
         instance = serializer.save()
+        task_type = None
         if instance.source.name == "chipexo_pugh_allevents":
-            chipexo_pugh_allevents_promoter_sig.delay(instance.id, self.request.user.id)
+            task_type = "chipexo_promoter_sig"
+        elif instance.source.name == "callingcards":
+            task_type = "callingcards_promoter_sig"
+
+        if task_type:
+            lock_id = f"add_data_lock"
+            acquire_lock = lambda: cache.add(lock_id, True, timeout=60 * 60)
+            release_lock = lambda: cache.delete(lock_id)
+
+            if acquire_lock():
+                try:
+                    # Create a chain of tasks
+                    task = chain(
+                        promoter_significance_task.s(instance.id, self.request.user.id, task_type),
+                        rank_response_tasks.s(user_id=self.request.user.id),
+                    )
+                    task.apply_async()
+                finally:
+                    release_lock()

yeastregulatorydb/regulatory_data/api/views/ExpressionViewSet.py

@@ -1,11 +1,14 @@
+from celery import chain
+from django.core.cache import cache
 from django_filters.rest_framework import DjangoFilterBackend
 from rest_framework import viewsets
 from rest_framework.authentication import SessionAuthentication, TokenAuthentication
 from rest_framework.permissions import IsAuthenticated
 
-from ...models.Expression import Expression
-from ..filters.ExpressionFilter import ExpressionFilter
-from ..serializers.ExpressionSerializer import ExpressionSerializer
+from ...models import Expression, PromoterSetSig
+from ...tasks import rank_response_task
+from ..filters import ExpressionFilter
+from ..serializers import ExpressionSerializer
 from .mixins.BulkUploadMixin import BulkUploadMixin
 from .mixins.UpdateModifiedMixin import UpdateModifiedMixin
 
@@ -21,3 +24,22 @@ class ExpressionViewSet(UpdateModifiedMixin, viewsets.ModelViewSet, BulkUploadMi
     serializer_class = ExpressionSerializer
     filter_backends = [DjangoFilterBackend]
     filterset_class = ExpressionFilter
+
+    def perform_create(self, serializer):
+        instance = serializer.save()
+        promotersetsig_id_list = PromoterSetSig.objects.filter(
+            binding__regulator__id=instance.regulator.id
+        ).values_list("id", flat=True)
+
+        # Create a chain of tasks for each promotersetsig_id
+        lock_id = f"add_data_lock"
+        acquire_lock = lambda: cache.add(lock_id, True, timeout=60 * 60)
+        release_lock = lambda: cache.delete(lock_id)
+
+        if acquire_lock():
+            try:
+                # Create a chain of tasks
+                for promotersetsig_id in promotersetsig_id_list:
+                    rank_response_task.apply_async(args=[promotersetsig_id, self.request.user.id])
+            finally:
+                release_lock()

yeastregulatorydb/regulatory_data/api/views/PromoterSetViewSet.py

@@ -1,9 +1,12 @@
+from celery import chain
+from django.core.cache import cache
 from django_filters.rest_framework import DjangoFilterBackend
 from rest_framework import viewsets
 from rest_framework.authentication import SessionAuthentication, TokenAuthentication
 from rest_framework.permissions import IsAuthenticated
 
-from ...models.PromoterSet import PromoterSet
+from ...models import Binding, PromoterSet
+from ...tasks import promoter_significance_task, rank_response_tasks
 from ..filters.PromoterSetFilter import PromoterSetFilter
 from ..serializers.PromoterSetSerializer import PromoterSetSerializer
 from .mixins.UpdateModifiedMixin import UpdateModifiedMixin
@@ -20,3 +23,23 @@ class PromoterSetViewSet(UpdateModifiedMixin, viewsets.ModelViewSet):
     serializer_class = PromoterSetSerializer
     filter_backends = [DjangoFilterBackend]
     filterset_class = PromoterSetFilter
+
+
+def perform_create(self, serializer):
+    instance = serializer.save()
+
+    lock_id = f"add_data_lock"
+    acquire_lock = lambda: cache.add(lock_id, True, timeout=60 * 60)
+    release_lock = lambda: cache.delete(lock_id)
+
+    if acquire_lock():
+        try:
+            for binding_obj in Binding.objects.all():
+                # Call promoter_significance_task and then rank_response_tasks
+                task = chain(
+                    promoter_significance_task.s(binding_obj.id, self.request.user.id, instance.id),
+                    rank_response_tasks.s(user_id=self.request.user.id),
+                )
+                task.apply_async()
+        finally:
+            release_lock()

yeastregulatorydb/regulatory_data/api/views/RankResponseViewSet.py

@@ -0,0 +1,57 @@
+import tempfile
+
+import pandas as pd
+from django.http import HttpResponse
+from django_filters.rest_framework import DjangoFilterBackend
+from rest_framework import status, viewsets
+from rest_framework.authentication import SessionAuthentication, TokenAuthentication
+from rest_framework.decorators import action
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
+
+from ...models import RankResponse
+from ...utils.extract_file_from_storage import extract_file_from_storage
+from ..filters.RankResponseFilter import RankResponseFilter
+from ..serializers.RankResponseSerializer import RankResponseSerializer
+from .mixins.UpdateModifiedMixin import UpdateModifiedMixin
+
+
+class RankResponseViewSet(UpdateModifiedMixin, viewsets.ModelViewSet):
+    """
+    A viewset for viewing and editing RankResponse instances.
+    """
+
+    queryset = RankResponse.objects.all()
+    authentication_classes = [SessionAuthentication, TokenAuthentication]
+    permission_classes = [IsAuthenticated]
+    serializer_class = RankResponseSerializer
+    filter_backends = [DjangoFilterBackend]
+    filterset_class = RankResponseFilter
+
+    @action(detail=False, methods=["get"])
+    def summary(self, request, *args, **kwargs):
+        if "rank_response_id" not in request.query_params:
+            return Response({"error": "rank_response_id must be specified"}, status=status.HTTP_400_BAD_REQUEST)
+
+        filtered_queryset = self.filter_queryset(self.get_queryset())
+        if len(filtered_queryset) != 1:
+            return Response(
+                {
+                    "error": "rank_response_summary returned multiple matches to your query. There should only be 1. "
+                    "Contact our admin -- this message shouldn't appear -- but in the meantime, "
+                    "re-submit with only the `rank_response_id` as a paramater"
+                },
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+        with tempfile.TemporaryDirectory() as tmpdir:
+            for rank_response_record in filtered_queryset:
+                # Iterate over the filtered queryset
+                filepath = extract_file_from_storage(rank_response_record.file, tmpdir)
+                df = pd.read_csv(filepath, compression="gzip")
+
+            with tempfile.NamedTemporaryFile(suffix=".gz") as tmpfile:
+                df.to_csv(tmpfile.name, compression="gzip", index=False)
+                tmpfile.seek(0)
+                response = HttpResponse(tmpfile, content_type="application/gzip")
+                response["Content-Disposition"] = "attachment; filename=rank_response_summary.csvgz"
+                return response