Merge pull request #6415 from chanzuckerberg/staging

chore: prod deploy 1/4/24
chanzuckerberg · Jan 4, 2024 · 8fc12ae · 8fc12ae
2 parents 9cd4be9 + 49a433d
commit 8fc12ae
Show file tree

Hide file tree

Showing 25 changed files with 156 additions and 661 deletions.
diff --git a/.happy/terraform/modules/batch/main.tf b/.happy/terraform/modules/batch/main.tf
@@ -119,70 +119,6 @@ resource aws_batch_job_definition dataset_metadata_update {
 })
 }
 
-resource aws_batch_job_definition reprocess_dataset_metadata {
-  # this was used to reprocess dataset metadata in place when an error was found after publishing cellxgene schema 4.0
-  # TODO: can be removed after 4.0 migration is complete
-  type = "container"
-  name = "dp-${var.deployment_stage}-${var.custom_stack_name}-reprocess-dataset-metadata"
-  container_properties = jsonencode({
-  "command": ["python3", "-m", "backend.layers.processing.reprocess_dataset_metadata"],
-  "jobRoleArn": "${var.batch_role_arn}",
-  "image": "${var.image}",
-  "memory": var.batch_container_memory_limit,
-  "environment": [
-    {
-      "name": "ARTIFACT_BUCKET",
-      "value": "${var.artifact_bucket}"
-    },
-    {
-      "name": "CELLXGENE_BUCKET",
-      "value": "${var.cellxgene_bucket}"
-    },
-    {
-      "name": "DATASETS_BUCKET",
-      "value": "${var.datasets_bucket}"
-    },
-    {
-      "name": "DEPLOYMENT_STAGE",
-      "value": "${var.deployment_stage}"
-    },
-    {
-      "name": "AWS_DEFAULT_REGION",
-      "value": "${data.aws_region.current.name}"
-    },
-    {
-      "name": "REMOTE_DEV_PREFIX",
-      "value": "${var.remote_dev_prefix}"
-    }
-  ],
-  "vcpus": 8,
-  "linuxParameters": {
-     "maxSwap": 800000,
-     "swappiness": 60
-  },
-    "retryStrategy": {
-    "attempts": 3,
-    "evaluateOnExit": [
-      {
-          "action": "RETRY",
-          "onReason": "Task failed to start"
-      },
-      {
-          "action": "EXIT",
-          "onReason": "*"
-      }
-    ]
-  },
-  "logConfiguration": {
-    "logDriver": "awslogs",
-    "options": {
-      "awslogs-group": "${aws_cloudwatch_log_group.cloud_watch_logs_group.id}",
-      "awslogs-region": "${data.aws_region.current.name}"
-    }
-  }
-})
-}
-
 resource aws_cloudwatch_log_group cloud_watch_logs_group {
   retention_in_days = 365
   name              = "/dp/${var.deployment_stage}/${var.custom_stack_name}/upload"

diff --git a/backend/common/feature_flag.py b/backend/common/feature_flag.py
@@ -11,26 +11,26 @@
 
 To use a feature flag:
 ```
-if FeatureFlagService.is_enabled(FeatureFlagValues.SCHEMA_4):
-  <logic that only applies if schema 4 is enabled>
+if FeatureFlagService.is_enabled(FeatureFlagValues.<feature_flag_value_name>):
+  <logic that only applies if feature flag is enabled>
 ```
 
 To mock a feature flag in a test:
 ```
 self.mock_config = CorporaConfig()
-self.mock_config.set(dict(schema_4_feature_flag="True"))
+self.mock_config.set(dict(<feature_flag_value_name>="True"))
 ```
 """
 
-FeatureFlag = Literal["schema_4_feature_flag", "citation_update_feature_flag"]
+FeatureFlag = Literal["citation_update_feature_flag"]
 
 
 class FeatureFlagValues:
-    SCHEMA_4 = "schema_4_feature_flag"
     CITATION_UPDATE = "citation_update_feature_flag"
 
 
 class FeatureFlagService:
+    @staticmethod
     def is_enabled(feature_flag: FeatureFlag) -> bool:
         flag_value = getattr(CorporaConfig(), feature_flag, "").lower()
         return flag_value == "true"
diff --git a/backend/common/providers/crossref_provider.py b/backend/common/providers/crossref_provider.py
diff --git a/backend/curation/api/v1/curation/collections/common.py b/backend/curation/api/v1/curation/collections/common.py
@@ -4,7 +4,6 @@
 from uuid import UUID
 
 from backend.common.corpora_config import CorporaConfig
-from backend.common.feature_flag import FeatureFlagService, FeatureFlagValues
 from backend.common.utils.http_exceptions import ForbiddenHTTPException, GoneHTTPException, NotFoundHTTPException
 from backend.layers.auth.user_info import UserInfo
 from backend.layers.business.business import BusinessLogic
@@ -228,10 +227,6 @@ def reshape_dataset_for_curation_api(
             if col is not None:
                 ds[column] = col
 
-    if ds.get("tissue") is not None and not FeatureFlagService.is_enabled(FeatureFlagValues.SCHEMA_4):
-        for tissue in ds["tissue"]:
-            del tissue["tissue_type"]
-
     ds["dataset_id"] = dataset_version.dataset_id.id
     ds["dataset_version_id"] = dataset_version.version_id.id
     # Get none preview specific dataset fields

diff --git a/backend/layers/processing/process_validate.py b/backend/layers/processing/process_validate.py
@@ -3,7 +3,6 @@
 import numpy
 import scanpy
 
-from backend.common.feature_flag import FeatureFlagService, FeatureFlagValues
 from backend.common.utils.corpora_constants import CorporaConstants
 from backend.layers.business.business_interface import BusinessLogicInterface
 from backend.layers.common.entities import (
@@ -82,8 +81,7 @@ def validate_h5ad_file_and_add_labels(
         if not is_valid:
             raise ValidationFailed(errors)
         else:
-            if FeatureFlagService.is_enabled(FeatureFlagValues.SCHEMA_4):
-                self.populate_dataset_citation(collection_version_id, dataset_version_id, output_filename)
+            self.populate_dataset_citation(collection_version_id, dataset_version_id, output_filename)
 
             # TODO: optionally, these could be batched into one
             self.update_processing_status(dataset_version_id, DatasetStatusKey.H5AD, DatasetConversionStatus.CONVERTED)
@@ -172,9 +170,7 @@ def _get_batch_condition() -> Optional[str]:
         return DatasetMetadata(
             name=adata.uns["title"],
             organism=_get_term_pairs("organism"),
-            tissue=_get_tissue_terms()
-            if FeatureFlagService.is_enabled(FeatureFlagValues.SCHEMA_4)
-            else _get_term_pairs("tissue"),
+            tissue=_get_tissue_terms(),
             assay=_get_term_pairs("assay"),
             disease=_get_term_pairs("disease"),
             sex=_get_term_pairs("sex"),