From 2b834fb780885a837d37990f6603e2ee8aa628e0 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 8 Jan 2025 07:24:53 -0800 Subject: [PATCH 1/2] remove replace table test --- tests/test_store.py | 93 --------------------------------------------- 1 file changed, 93 deletions(-) diff --git a/tests/test_store.py b/tests/test_store.py index 49b79a3c0..e73c64072 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1139,99 +1139,6 @@ def test_tidy_table_no_manifest_uploaded(self, synapse_store): @pytest.mark.table_operations class TestTableOperations: - @pytest.mark.slow_test - @pytest.mark.parametrize( - "table_column_names", - ["display_name", "display_label", "class_label"], - ids=["tcn_display_name", "tcn_display_label", "tcn_class_label"], - ) - @pytest.mark.parametrize( - "annotation_keys", - ["display_label", "class_label"], - ids=["aks_display_label", "aks_class_label"], - ) - async def test_create_table( - self, - helpers: Helpers, - synapse_store: SynapseStorage, - projectId: str, - datasetId: str, - table_column_names: str, - annotation_keys: str, - dmge: DataModelGraphExplorer, - schedule_for_cleanup: Callable[[CleanupItem], None], - ) -> None: - # GIVEN a table to create - table_manipulation = None - table_name = f"followup_synapse_storage_manifest_table_integration_test_{str(uuid.uuid4()).replace('-', '_')}" - schedule_for_cleanup(CleanupItem(name=table_name, parent_id=projectId)) - - # AND a manifest to associate metadata with files - manifest_path = "mock_manifests/table_manifest.csv" - - # AND a copy of all the folders in the manifest. Added to the dataset directory for easy cleanup - manifest = helpers.get_data_frame(manifest_path) - - async def copy_folder_and_update_manifest( - row: pd.Series, - index: int, - datasetId: str, - synapse_store: SynapseStorage, - manifest: pd.DataFrame, - schedule_for_cleanup: Callable[[CleanupItem], None], - ) -> None: - """Internal function to copy a folder and update the manifest.""" - folder_id = row["entityId"] - folder_copy = await FolderModel(id=folder_id).copy_async( - parent_id=datasetId, synapse_client=synapse_store.syn - ) - schedule_for_cleanup(CleanupItem(synapse_id=folder_copy.id)) - manifest.at[index, "entityId"] = folder_copy.id - - tasks = [] - - for index, row in manifest.iterrows(): - tasks.append( - copy_folder_and_update_manifest( - row, index, datasetId, synapse_store, manifest, schedule_for_cleanup - ) - ) - await asyncio.gather(*tasks) - - with patch.object( - synapse_store, "_generate_table_name", return_value=(table_name, "followup") - ), patch.object( - synapse_store, "getDatasetProject", return_value=projectId - ), tempfile.NamedTemporaryFile( - delete=True, - suffix=".csv", - dir=create_temp_folder(path=tempfile.gettempdir()), - ) as tmp_file: - # Write the DF to a temporary file to prevent modifying the original - manifest.to_csv(tmp_file.name, index=False) - - # WHEN I associate metadata with files - manifest_id = synapse_store.associateMetadataWithFiles( - dmge=dmge, - metadataManifestPath=tmp_file.name, - datasetId=datasetId, - manifest_record_type="table_and_file", - hideBlanks=True, - restrict_manifest=False, - table_manipulation=table_manipulation, - table_column_names=table_column_names, - annotation_keys=annotation_keys, - ) - schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) - - # THEN the table should exist - existing_table_id = synapse_store.syn.findEntityId( - name=table_name, parent=projectId - ) - - # assert table exists - assert existing_table_id is not None - @pytest.mark.slow_test @pytest.mark.parametrize( "table_column_names", From 6811c6f7bcfe1b8b45c8bfee7945f79561d4f850 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 8 Jan 2025 07:25:26 -0800 Subject: [PATCH 2/2] reduce number of folders to copy in test manifest --- tests/data/mock_manifests/table_manifest.csv | 7 ------- tests/data/mock_manifests/table_manifest_replacement.csv | 7 ------- 2 files changed, 14 deletions(-) diff --git a/tests/data/mock_manifests/table_manifest.csv b/tests/data/mock_manifests/table_manifest.csv index c64988577..f0d07db9b 100644 --- a/tests/data/mock_manifests/table_manifest.csv +++ b/tests/data/mock_manifests/table_manifest.csv @@ -1,10 +1,3 @@ Component,Days to Follow Up,Adverse Event,Progression or Recurrence,Barretts Esophagus Goblet Cells Present,BMI,Cause of Response,Comorbidity,Comorbidity Method of Diagnosis,Days to Adverse Event,Days to Comorbidity,Diabetes Treatment Type,Disease Response,DLCO Ref Predictive Percent,ECOG Performance Status,FEV1 FVC Post Bronch Percent,FEV 1 FVC Pre Bronch Percent,FEV1 Ref Post Bronch Percent,FEV1 Ref Pre Bronch Percent,Height,Hepatitis Sustained Virological Response,HPV Positive Type,Karnofsky Performance Status,Menopause Status,Pancreatitis Onset Year,Reflux Treatment Type,Risk Factor,Risk Factor Treatment,Viral Hepatitis Serologies,Weight,Days to Progression,Days to Progression Free,Days to Recurrence,Progression or Recurrence Anatomic Site,Progression or Recurrence Type,entityId,HTAN Participant ID FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249015,455.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249915,456.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22687161,457.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22688035,458.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978930,459.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978946,460.0 FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978975,461.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22979177,462.0 -FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn21989551,454.0 diff --git a/tests/data/mock_manifests/table_manifest_replacement.csv b/tests/data/mock_manifests/table_manifest_replacement.csv index 71fd13e30..fea37d859 100644 --- a/tests/data/mock_manifests/table_manifest_replacement.csv +++ b/tests/data/mock_manifests/table_manifest_replacement.csv @@ -1,10 +1,3 @@ Component,Days to Follow Up,Adverse Event,Progression or Recurrence,Barretts Esophagus Goblet Cells Present,BMI,Cause of Response,Comorbidity,Comorbidity Method of Diagnosis,Days to Adverse Event,Days to Comorbidity,Diabetes Treatment Type,Disease Response,DLCO Ref Predictive Percent,ECOG Performance Status,FEV1 FVC Post Bronch Percent,FEV 1 FVC Pre Bronch Percent,FEV1 Ref Post Bronch Percent,FEV1 Ref Pre Bronch Percent,Height,Hepatitis Sustained Virological Response,HPV Positive Type,Karnofsky Performance Status,Menopause Status,Pancreatitis Onset Year,Reflux Treatment Type,Risk Factor,Risk Factor Treatment,Viral Hepatitis Serologies,Weight,Days to Progression,Days to Progression Free,Days to Recurrence,Progression or Recurrence Anatomic Site,Progression or Recurrence Type,entityId,HTAN Participant ID,Uuid FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249015,455,318e2d8c-83be-4b19-9a9f-5e97ce9c9cf7 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249915,456,600666ee-1f96-49f8-b848-2be751cb59f9 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22687161,457,1fb32068-7b6c-4d12-8031-96d91229dda0 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22688035,458,964949f3-efe1-47cd-8113-dedb909fd312 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978930,459,c562590f-359a-4ff5-bbd7-47e7ad840a32 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978946,460,a77e0882-7b6f-42a9-9e60-c4d2bfd72bb7 FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978975,461,e7705ed5-ade4-4451-ae26-9c790e35e878 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22979177,462,2897b73c-e225-4dc3-a093-96f645332ea3 -FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn21989551,454,26a1cdba-dd37-483a-be6e-d1a4f5a9521b