Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SCHEMATIC-208] fix slow test times #1566

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

[SCHEMATIC-208] fix slow test times #1566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2b834fb
remove replace table test
andrewelamb Jan 8, 2025
6811c6f
reduce number of folders to copy in test manifest
andrewelamb Jan 8, 2025
c9898b1
Merge branch 'develop' into SCHEMATIC-208-test-times
andrewelamb Jan 8, 2025
f25fbdb
Merge branch 'develop' into SCHEMATIC-208-test-times
andrewelamb Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions tests/data/mock_manifests/table_manifest.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,3 @@
Component,Days to Follow Up,Adverse Event,Progression or Recurrence,Barretts Esophagus Goblet Cells Present,BMI,Cause of Response,Comorbidity,Comorbidity Method of Diagnosis,Days to Adverse Event,Days to Comorbidity,Diabetes Treatment Type,Disease Response,DLCO Ref Predictive Percent,ECOG Performance Status,FEV1 FVC Post Bronch Percent,FEV 1 FVC Pre Bronch Percent,FEV1 Ref Post Bronch Percent,FEV1 Ref Pre Bronch Percent,Height,Hepatitis Sustained Virological Response,HPV Positive Type,Karnofsky Performance Status,Menopause Status,Pancreatitis Onset Year,Reflux Treatment Type,Risk Factor,Risk Factor Treatment,Viral Hepatitis Serologies,Weight,Days to Progression,Days to Progression Free,Days to Recurrence,Progression or Recurrence Anatomic Site,Progression or Recurrence Type,entityId,HTAN Participant ID
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249015,455.0
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249915,456.0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Were these entities deleted on synapse?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They were not, they are in an HTAN project.

FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22687161,457.0
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22688035,458.0
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978930,459.0
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978946,460.0
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978975,461.0
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22979177,462.0
FollowUp,73.0,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn21989551,454.0
7 changes: 0 additions & 7 deletions tests/data/mock_manifests/table_manifest_replacement.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,3 @@
Component,Days to Follow Up,Adverse Event,Progression or Recurrence,Barretts Esophagus Goblet Cells Present,BMI,Cause of Response,Comorbidity,Comorbidity Method of Diagnosis,Days to Adverse Event,Days to Comorbidity,Diabetes Treatment Type,Disease Response,DLCO Ref Predictive Percent,ECOG Performance Status,FEV1 FVC Post Bronch Percent,FEV 1 FVC Pre Bronch Percent,FEV1 Ref Post Bronch Percent,FEV1 Ref Pre Bronch Percent,Height,Hepatitis Sustained Virological Response,HPV Positive Type,Karnofsky Performance Status,Menopause Status,Pancreatitis Onset Year,Reflux Treatment Type,Risk Factor,Risk Factor Treatment,Viral Hepatitis Serologies,Weight,Days to Progression,Days to Progression Free,Days to Recurrence,Progression or Recurrence Anatomic Site,Progression or Recurrence Type,entityId,HTAN Participant ID,Uuid
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249015,455,318e2d8c-83be-4b19-9a9f-5e97ce9c9cf7
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22249915,456,600666ee-1f96-49f8-b848-2be751cb59f9
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22687161,457,1fb32068-7b6c-4d12-8031-96d91229dda0
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22688035,458,964949f3-efe1-47cd-8113-dedb909fd312
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978930,459,c562590f-359a-4ff5-bbd7-47e7ad840a32
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978946,460,a77e0882-7b6f-42a9-9e60-c4d2bfd72bb7
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22978975,461,e7705ed5-ade4-4451-ae26-9c790e35e878
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn22979177,462,2897b73c-e225-4dc3-a093-96f645332ea3
FollowUp,89,,Not Reported,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,syn21989551,454,26a1cdba-dd37-483a-be6e-d1a4f5a9521b
93 changes: 0 additions & 93 deletions tests/test_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1139,99 +1139,6 @@ def test_tidy_table_no_manifest_uploaded(self, synapse_store):

@pytest.mark.table_operations
class TestTableOperations:
@pytest.mark.slow_test
@pytest.mark.parametrize(
"table_column_names",
["display_name", "display_label", "class_label"],
ids=["tcn_display_name", "tcn_display_label", "tcn_class_label"],
)
@pytest.mark.parametrize(
"annotation_keys",
["display_label", "class_label"],
ids=["aks_display_label", "aks_class_label"],
)
async def test_create_table(
self,
helpers: Helpers,
synapse_store: SynapseStorage,
projectId: str,
datasetId: str,
table_column_names: str,
annotation_keys: str,
dmge: DataModelGraphExplorer,
schedule_for_cleanup: Callable[[CleanupItem], None],
) -> None:
# GIVEN a table to create
table_manipulation = None
table_name = f"followup_synapse_storage_manifest_table_integration_test_{str(uuid.uuid4()).replace('-', '_')}"
schedule_for_cleanup(CleanupItem(name=table_name, parent_id=projectId))

# AND a manifest to associate metadata with files
manifest_path = "mock_manifests/table_manifest.csv"

# AND a copy of all the folders in the manifest. Added to the dataset directory for easy cleanup
manifest = helpers.get_data_frame(manifest_path)

async def copy_folder_and_update_manifest(
row: pd.Series,
index: int,
datasetId: str,
synapse_store: SynapseStorage,
manifest: pd.DataFrame,
schedule_for_cleanup: Callable[[CleanupItem], None],
) -> None:
"""Internal function to copy a folder and update the manifest."""
folder_id = row["entityId"]
folder_copy = await FolderModel(id=folder_id).copy_async(
parent_id=datasetId, synapse_client=synapse_store.syn
)
schedule_for_cleanup(CleanupItem(synapse_id=folder_copy.id))
manifest.at[index, "entityId"] = folder_copy.id

tasks = []

for index, row in manifest.iterrows():
tasks.append(
copy_folder_and_update_manifest(
row, index, datasetId, synapse_store, manifest, schedule_for_cleanup
)
)
await asyncio.gather(*tasks)

with patch.object(
synapse_store, "_generate_table_name", return_value=(table_name, "followup")
), patch.object(
synapse_store, "getDatasetProject", return_value=projectId
), tempfile.NamedTemporaryFile(
delete=True,
suffix=".csv",
dir=create_temp_folder(path=tempfile.gettempdir()),
) as tmp_file:
# Write the DF to a temporary file to prevent modifying the original
manifest.to_csv(tmp_file.name, index=False)

# WHEN I associate metadata with files
manifest_id = synapse_store.associateMetadataWithFiles(
dmge=dmge,
metadataManifestPath=tmp_file.name,
datasetId=datasetId,
manifest_record_type="table_and_file",
hideBlanks=True,
restrict_manifest=False,
table_manipulation=table_manipulation,
table_column_names=table_column_names,
annotation_keys=annotation_keys,
)
schedule_for_cleanup(CleanupItem(synapse_id=manifest_id))

# THEN the table should exist
existing_table_id = synapse_store.syn.findEntityId(
name=table_name, parent=projectId
)

# assert table exists
assert existing_table_id is not None

@pytest.mark.slow_test
@pytest.mark.parametrize(
"table_column_names",
Expand Down
Loading