diff --git a/poetry.lock b/poetry.lock index 25750a1441..87e5eadcf1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,6 +12,18 @@ files = [ {file = "apipkg-3.0.1.tar.gz", hash = "sha256:f8c021adafc9132ac2fba9fd3c5768365d0a8c10aa375fb15e329f1fce8a5f01"}, ] +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] + [[package]] name = "aws-requests-auth" version = "0.4.3" @@ -29,18 +41,18 @@ requests = ">=0.14.0" [[package]] name = "awscli" -version = "1.27.117" +version = "1.27.135" description = "Universal Command Line Environment for AWS." category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "awscli-1.27.117-py3-none-any.whl", hash = "sha256:e107db89b8e0e969ff9134fd95d4992d62c188a05406654b00d924ecd3cb6f1b"}, - {file = "awscli-1.27.117.tar.gz", hash = "sha256:0611ae60aa486dc4661753ff91e42ed9daa43fddc2be13492ff64e52c9f12851"}, + {file = "awscli-1.27.135-py3-none-any.whl", hash = "sha256:cc103cc20828309cbb95349a2e3d3b48457a5bd2e92e366bc9cf3bd67ebafe3e"}, + {file = "awscli-1.27.135.tar.gz", hash = "sha256:ab09ae685eca061c6cc36c15e2dc9d28bea18b448d414453aa95f88450a4a802"}, ] [package.dependencies] -botocore = "1.29.117" +botocore = "1.29.135" colorama = ">=0.2.5,<0.4.5" docutils = ">=0.10,<0.17" PyYAML = ">=3.10,<5.5" @@ -165,18 +177,18 @@ files = [ [[package]] name = "boto3" -version = "1.26.117" +version = "1.26.135" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.26.117-py3-none-any.whl", hash = "sha256:84cc8947560d89581ef693895cb36d8c8c70d567900d86c8f7cc612d037fdd17"}, - {file = "boto3-1.26.117.tar.gz", hash = "sha256:1e9128b7d8a1b8af789ea6a04a46e641d82446e1e15c022c0fef697d3c9aab97"}, + {file = "boto3-1.26.135-py3-none-any.whl", hash = "sha256:ba7ca9215a1026620741273da10d0d3cceb9f7649f7c101e616a287071826f9d"}, + {file = "boto3-1.26.135.tar.gz", hash = "sha256:23523d5d6aa51bba2461d67f6eb458d83b6a52d18e3d953b1ce71209b66462ec"}, ] [package.dependencies] -botocore = ">=1.29.117,<1.30.0" +botocore = ">=1.29.135,<1.30.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -185,14 +197,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "boto3-stubs" -version = "1.26.117" -description = "Type annotations for boto3 1.26.117 generated with mypy-boto3-builder 7.14.5" +version = "1.26.135" +description = "Type annotations for boto3 1.26.135 generated with mypy-boto3-builder 7.14.5" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "boto3-stubs-1.26.117.tar.gz", hash = "sha256:5bbcd0446463023415d4e2b130610cc9b8a01e244733ab27efe5272faef1cb7d"}, - {file = "boto3_stubs-1.26.117-py3-none-any.whl", hash = "sha256:b57274ad5e7183e5247133299ef19ffabd8b0296250339f7481a7c355d464d18"}, + {file = "boto3-stubs-1.26.135.tar.gz", hash = "sha256:d4be8288892056725a37d87ed07062d6acc34af59f2fead9eb30a8d214406d5a"}, + {file = "boto3_stubs-1.26.135-py3-none-any.whl", hash = "sha256:c3cfafeb34a6443dee8066924023f476a8919e3514343623b3d8de2a8f842a3b"}, ] [package.dependencies] @@ -206,7 +218,7 @@ account = ["mypy-boto3-account (>=1.26.0,<1.27.0)"] acm = ["mypy-boto3-acm (>=1.26.0,<1.27.0)"] acm-pca = ["mypy-boto3-acm-pca (>=1.26.0,<1.27.0)"] alexaforbusiness = ["mypy-boto3-alexaforbusiness (>=1.26.0,<1.27.0)"] -all = ["mypy-boto3-accessanalyzer (>=1.26.0,<1.27.0)", "mypy-boto3-account (>=1.26.0,<1.27.0)", "mypy-boto3-acm (>=1.26.0,<1.27.0)", "mypy-boto3-acm-pca (>=1.26.0,<1.27.0)", "mypy-boto3-alexaforbusiness (>=1.26.0,<1.27.0)", "mypy-boto3-amp (>=1.26.0,<1.27.0)", "mypy-boto3-amplify (>=1.26.0,<1.27.0)", "mypy-boto3-amplifybackend (>=1.26.0,<1.27.0)", "mypy-boto3-amplifyuibuilder (>=1.26.0,<1.27.0)", "mypy-boto3-apigateway (>=1.26.0,<1.27.0)", "mypy-boto3-apigatewaymanagementapi (>=1.26.0,<1.27.0)", "mypy-boto3-apigatewayv2 (>=1.26.0,<1.27.0)", "mypy-boto3-appconfig (>=1.26.0,<1.27.0)", "mypy-boto3-appconfigdata (>=1.26.0,<1.27.0)", "mypy-boto3-appflow (>=1.26.0,<1.27.0)", "mypy-boto3-appintegrations (>=1.26.0,<1.27.0)", "mypy-boto3-application-autoscaling (>=1.26.0,<1.27.0)", "mypy-boto3-application-insights (>=1.26.0,<1.27.0)", "mypy-boto3-applicationcostprofiler (>=1.26.0,<1.27.0)", "mypy-boto3-appmesh (>=1.26.0,<1.27.0)", "mypy-boto3-apprunner (>=1.26.0,<1.27.0)", "mypy-boto3-appstream (>=1.26.0,<1.27.0)", "mypy-boto3-appsync (>=1.26.0,<1.27.0)", "mypy-boto3-arc-zonal-shift (>=1.26.0,<1.27.0)", "mypy-boto3-athena (>=1.26.0,<1.27.0)", "mypy-boto3-auditmanager (>=1.26.0,<1.27.0)", "mypy-boto3-autoscaling (>=1.26.0,<1.27.0)", "mypy-boto3-autoscaling-plans (>=1.26.0,<1.27.0)", "mypy-boto3-backup (>=1.26.0,<1.27.0)", "mypy-boto3-backup-gateway (>=1.26.0,<1.27.0)", "mypy-boto3-backupstorage (>=1.26.0,<1.27.0)", "mypy-boto3-batch (>=1.26.0,<1.27.0)", "mypy-boto3-billingconductor (>=1.26.0,<1.27.0)", "mypy-boto3-braket (>=1.26.0,<1.27.0)", "mypy-boto3-budgets (>=1.26.0,<1.27.0)", "mypy-boto3-ce (>=1.26.0,<1.27.0)", "mypy-boto3-chime (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-identity (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-media-pipelines (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-meetings (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-messaging (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-voice (>=1.26.0,<1.27.0)", "mypy-boto3-cleanrooms (>=1.26.0,<1.27.0)", "mypy-boto3-cloud9 (>=1.26.0,<1.27.0)", "mypy-boto3-cloudcontrol (>=1.26.0,<1.27.0)", "mypy-boto3-clouddirectory (>=1.26.0,<1.27.0)", "mypy-boto3-cloudformation (>=1.26.0,<1.27.0)", "mypy-boto3-cloudfront (>=1.26.0,<1.27.0)", "mypy-boto3-cloudhsm (>=1.26.0,<1.27.0)", "mypy-boto3-cloudhsmv2 (>=1.26.0,<1.27.0)", "mypy-boto3-cloudsearch (>=1.26.0,<1.27.0)", "mypy-boto3-cloudsearchdomain (>=1.26.0,<1.27.0)", "mypy-boto3-cloudtrail (>=1.26.0,<1.27.0)", "mypy-boto3-cloudtrail-data (>=1.26.0,<1.27.0)", "mypy-boto3-cloudwatch (>=1.26.0,<1.27.0)", "mypy-boto3-codeartifact (>=1.26.0,<1.27.0)", "mypy-boto3-codebuild (>=1.26.0,<1.27.0)", "mypy-boto3-codecatalyst (>=1.26.0,<1.27.0)", "mypy-boto3-codecommit (>=1.26.0,<1.27.0)", "mypy-boto3-codedeploy (>=1.26.0,<1.27.0)", "mypy-boto3-codeguru-reviewer (>=1.26.0,<1.27.0)", "mypy-boto3-codeguruprofiler (>=1.26.0,<1.27.0)", "mypy-boto3-codepipeline (>=1.26.0,<1.27.0)", "mypy-boto3-codestar (>=1.26.0,<1.27.0)", "mypy-boto3-codestar-connections (>=1.26.0,<1.27.0)", "mypy-boto3-codestar-notifications (>=1.26.0,<1.27.0)", "mypy-boto3-cognito-identity (>=1.26.0,<1.27.0)", "mypy-boto3-cognito-idp (>=1.26.0,<1.27.0)", "mypy-boto3-cognito-sync (>=1.26.0,<1.27.0)", "mypy-boto3-comprehend (>=1.26.0,<1.27.0)", "mypy-boto3-comprehendmedical (>=1.26.0,<1.27.0)", "mypy-boto3-compute-optimizer (>=1.26.0,<1.27.0)", "mypy-boto3-config (>=1.26.0,<1.27.0)", "mypy-boto3-connect (>=1.26.0,<1.27.0)", "mypy-boto3-connect-contact-lens (>=1.26.0,<1.27.0)", "mypy-boto3-connectcampaigns (>=1.26.0,<1.27.0)", "mypy-boto3-connectcases (>=1.26.0,<1.27.0)", "mypy-boto3-connectparticipant (>=1.26.0,<1.27.0)", "mypy-boto3-controltower (>=1.26.0,<1.27.0)", "mypy-boto3-cur (>=1.26.0,<1.27.0)", "mypy-boto3-customer-profiles (>=1.26.0,<1.27.0)", "mypy-boto3-databrew (>=1.26.0,<1.27.0)", "mypy-boto3-dataexchange (>=1.26.0,<1.27.0)", "mypy-boto3-datapipeline (>=1.26.0,<1.27.0)", "mypy-boto3-datasync (>=1.26.0,<1.27.0)", "mypy-boto3-dax (>=1.26.0,<1.27.0)", "mypy-boto3-detective (>=1.26.0,<1.27.0)", "mypy-boto3-devicefarm (>=1.26.0,<1.27.0)", "mypy-boto3-devops-guru (>=1.26.0,<1.27.0)", "mypy-boto3-directconnect (>=1.26.0,<1.27.0)", "mypy-boto3-discovery (>=1.26.0,<1.27.0)", "mypy-boto3-dlm (>=1.26.0,<1.27.0)", "mypy-boto3-dms (>=1.26.0,<1.27.0)", "mypy-boto3-docdb (>=1.26.0,<1.27.0)", "mypy-boto3-docdb-elastic (>=1.26.0,<1.27.0)", "mypy-boto3-drs (>=1.26.0,<1.27.0)", "mypy-boto3-ds (>=1.26.0,<1.27.0)", "mypy-boto3-dynamodb (>=1.26.0,<1.27.0)", "mypy-boto3-dynamodbstreams (>=1.26.0,<1.27.0)", "mypy-boto3-ebs (>=1.26.0,<1.27.0)", "mypy-boto3-ec2 (>=1.26.0,<1.27.0)", "mypy-boto3-ec2-instance-connect (>=1.26.0,<1.27.0)", "mypy-boto3-ecr (>=1.26.0,<1.27.0)", "mypy-boto3-ecr-public (>=1.26.0,<1.27.0)", "mypy-boto3-ecs (>=1.26.0,<1.27.0)", "mypy-boto3-efs (>=1.26.0,<1.27.0)", "mypy-boto3-eks (>=1.26.0,<1.27.0)", "mypy-boto3-elastic-inference (>=1.26.0,<1.27.0)", "mypy-boto3-elasticache (>=1.26.0,<1.27.0)", "mypy-boto3-elasticbeanstalk (>=1.26.0,<1.27.0)", "mypy-boto3-elastictranscoder (>=1.26.0,<1.27.0)", "mypy-boto3-elb (>=1.26.0,<1.27.0)", "mypy-boto3-elbv2 (>=1.26.0,<1.27.0)", "mypy-boto3-emr (>=1.26.0,<1.27.0)", "mypy-boto3-emr-containers (>=1.26.0,<1.27.0)", "mypy-boto3-emr-serverless (>=1.26.0,<1.27.0)", "mypy-boto3-es (>=1.26.0,<1.27.0)", "mypy-boto3-events (>=1.26.0,<1.27.0)", "mypy-boto3-evidently (>=1.26.0,<1.27.0)", "mypy-boto3-finspace (>=1.26.0,<1.27.0)", "mypy-boto3-finspace-data (>=1.26.0,<1.27.0)", "mypy-boto3-firehose (>=1.26.0,<1.27.0)", "mypy-boto3-fis (>=1.26.0,<1.27.0)", "mypy-boto3-fms (>=1.26.0,<1.27.0)", "mypy-boto3-forecast (>=1.26.0,<1.27.0)", "mypy-boto3-forecastquery (>=1.26.0,<1.27.0)", "mypy-boto3-frauddetector (>=1.26.0,<1.27.0)", "mypy-boto3-fsx (>=1.26.0,<1.27.0)", "mypy-boto3-gamelift (>=1.26.0,<1.27.0)", "mypy-boto3-gamesparks (>=1.26.0,<1.27.0)", "mypy-boto3-glacier (>=1.26.0,<1.27.0)", "mypy-boto3-globalaccelerator (>=1.26.0,<1.27.0)", "mypy-boto3-glue (>=1.26.0,<1.27.0)", "mypy-boto3-grafana (>=1.26.0,<1.27.0)", "mypy-boto3-greengrass (>=1.26.0,<1.27.0)", "mypy-boto3-greengrassv2 (>=1.26.0,<1.27.0)", "mypy-boto3-groundstation (>=1.26.0,<1.27.0)", "mypy-boto3-guardduty (>=1.26.0,<1.27.0)", "mypy-boto3-health (>=1.26.0,<1.27.0)", "mypy-boto3-healthlake (>=1.26.0,<1.27.0)", "mypy-boto3-honeycode (>=1.26.0,<1.27.0)", "mypy-boto3-iam (>=1.26.0,<1.27.0)", "mypy-boto3-identitystore (>=1.26.0,<1.27.0)", "mypy-boto3-imagebuilder (>=1.26.0,<1.27.0)", "mypy-boto3-importexport (>=1.26.0,<1.27.0)", "mypy-boto3-inspector (>=1.26.0,<1.27.0)", "mypy-boto3-inspector2 (>=1.26.0,<1.27.0)", "mypy-boto3-internetmonitor (>=1.26.0,<1.27.0)", "mypy-boto3-iot (>=1.26.0,<1.27.0)", "mypy-boto3-iot-data (>=1.26.0,<1.27.0)", "mypy-boto3-iot-jobs-data (>=1.26.0,<1.27.0)", "mypy-boto3-iot-roborunner (>=1.26.0,<1.27.0)", "mypy-boto3-iot1click-devices (>=1.26.0,<1.27.0)", "mypy-boto3-iot1click-projects (>=1.26.0,<1.27.0)", "mypy-boto3-iotanalytics (>=1.26.0,<1.27.0)", "mypy-boto3-iotdeviceadvisor (>=1.26.0,<1.27.0)", "mypy-boto3-iotevents (>=1.26.0,<1.27.0)", "mypy-boto3-iotevents-data (>=1.26.0,<1.27.0)", "mypy-boto3-iotfleethub (>=1.26.0,<1.27.0)", "mypy-boto3-iotfleetwise (>=1.26.0,<1.27.0)", "mypy-boto3-iotsecuretunneling (>=1.26.0,<1.27.0)", "mypy-boto3-iotsitewise (>=1.26.0,<1.27.0)", "mypy-boto3-iotthingsgraph (>=1.26.0,<1.27.0)", "mypy-boto3-iottwinmaker (>=1.26.0,<1.27.0)", "mypy-boto3-iotwireless (>=1.26.0,<1.27.0)", "mypy-boto3-ivs (>=1.26.0,<1.27.0)", "mypy-boto3-ivs-realtime (>=1.26.0,<1.27.0)", "mypy-boto3-ivschat (>=1.26.0,<1.27.0)", "mypy-boto3-kafka (>=1.26.0,<1.27.0)", "mypy-boto3-kafkaconnect (>=1.26.0,<1.27.0)", "mypy-boto3-kendra (>=1.26.0,<1.27.0)", "mypy-boto3-kendra-ranking (>=1.26.0,<1.27.0)", "mypy-boto3-keyspaces (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-archived-media (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-media (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-signaling (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-webrtc-storage (>=1.26.0,<1.27.0)", "mypy-boto3-kinesisanalytics (>=1.26.0,<1.27.0)", "mypy-boto3-kinesisanalyticsv2 (>=1.26.0,<1.27.0)", "mypy-boto3-kinesisvideo (>=1.26.0,<1.27.0)", "mypy-boto3-kms (>=1.26.0,<1.27.0)", "mypy-boto3-lakeformation (>=1.26.0,<1.27.0)", "mypy-boto3-lambda (>=1.26.0,<1.27.0)", "mypy-boto3-lex-models (>=1.26.0,<1.27.0)", "mypy-boto3-lex-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-lexv2-models (>=1.26.0,<1.27.0)", "mypy-boto3-lexv2-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-license-manager (>=1.26.0,<1.27.0)", "mypy-boto3-license-manager-linux-subscriptions (>=1.26.0,<1.27.0)", "mypy-boto3-license-manager-user-subscriptions (>=1.26.0,<1.27.0)", "mypy-boto3-lightsail (>=1.26.0,<1.27.0)", "mypy-boto3-location (>=1.26.0,<1.27.0)", "mypy-boto3-logs (>=1.26.0,<1.27.0)", "mypy-boto3-lookoutequipment (>=1.26.0,<1.27.0)", "mypy-boto3-lookoutmetrics (>=1.26.0,<1.27.0)", "mypy-boto3-lookoutvision (>=1.26.0,<1.27.0)", "mypy-boto3-m2 (>=1.26.0,<1.27.0)", "mypy-boto3-machinelearning (>=1.26.0,<1.27.0)", "mypy-boto3-macie (>=1.26.0,<1.27.0)", "mypy-boto3-macie2 (>=1.26.0,<1.27.0)", "mypy-boto3-managedblockchain (>=1.26.0,<1.27.0)", "mypy-boto3-marketplace-catalog (>=1.26.0,<1.27.0)", "mypy-boto3-marketplace-entitlement (>=1.26.0,<1.27.0)", "mypy-boto3-marketplacecommerceanalytics (>=1.26.0,<1.27.0)", "mypy-boto3-mediaconnect (>=1.26.0,<1.27.0)", "mypy-boto3-mediaconvert (>=1.26.0,<1.27.0)", "mypy-boto3-medialive (>=1.26.0,<1.27.0)", "mypy-boto3-mediapackage (>=1.26.0,<1.27.0)", "mypy-boto3-mediapackage-vod (>=1.26.0,<1.27.0)", "mypy-boto3-mediastore (>=1.26.0,<1.27.0)", "mypy-boto3-mediastore-data (>=1.26.0,<1.27.0)", "mypy-boto3-mediatailor (>=1.26.0,<1.27.0)", "mypy-boto3-memorydb (>=1.26.0,<1.27.0)", "mypy-boto3-meteringmarketplace (>=1.26.0,<1.27.0)", "mypy-boto3-mgh (>=1.26.0,<1.27.0)", "mypy-boto3-mgn (>=1.26.0,<1.27.0)", "mypy-boto3-migration-hub-refactor-spaces (>=1.26.0,<1.27.0)", "mypy-boto3-migrationhub-config (>=1.26.0,<1.27.0)", "mypy-boto3-migrationhuborchestrator (>=1.26.0,<1.27.0)", "mypy-boto3-migrationhubstrategy (>=1.26.0,<1.27.0)", "mypy-boto3-mobile (>=1.26.0,<1.27.0)", "mypy-boto3-mq (>=1.26.0,<1.27.0)", "mypy-boto3-mturk (>=1.26.0,<1.27.0)", "mypy-boto3-mwaa (>=1.26.0,<1.27.0)", "mypy-boto3-neptune (>=1.26.0,<1.27.0)", "mypy-boto3-network-firewall (>=1.26.0,<1.27.0)", "mypy-boto3-networkmanager (>=1.26.0,<1.27.0)", "mypy-boto3-nimble (>=1.26.0,<1.27.0)", "mypy-boto3-oam (>=1.26.0,<1.27.0)", "mypy-boto3-omics (>=1.26.0,<1.27.0)", "mypy-boto3-opensearch (>=1.26.0,<1.27.0)", "mypy-boto3-opensearchserverless (>=1.26.0,<1.27.0)", "mypy-boto3-opsworks (>=1.26.0,<1.27.0)", "mypy-boto3-opsworkscm (>=1.26.0,<1.27.0)", "mypy-boto3-organizations (>=1.26.0,<1.27.0)", "mypy-boto3-outposts (>=1.26.0,<1.27.0)", "mypy-boto3-panorama (>=1.26.0,<1.27.0)", "mypy-boto3-personalize (>=1.26.0,<1.27.0)", "mypy-boto3-personalize-events (>=1.26.0,<1.27.0)", "mypy-boto3-personalize-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-pi (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint-email (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint-sms-voice (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint-sms-voice-v2 (>=1.26.0,<1.27.0)", "mypy-boto3-pipes (>=1.26.0,<1.27.0)", "mypy-boto3-polly (>=1.26.0,<1.27.0)", "mypy-boto3-pricing (>=1.26.0,<1.27.0)", "mypy-boto3-privatenetworks (>=1.26.0,<1.27.0)", "mypy-boto3-proton (>=1.26.0,<1.27.0)", "mypy-boto3-qldb (>=1.26.0,<1.27.0)", "mypy-boto3-qldb-session (>=1.26.0,<1.27.0)", "mypy-boto3-quicksight (>=1.26.0,<1.27.0)", "mypy-boto3-ram (>=1.26.0,<1.27.0)", "mypy-boto3-rbin (>=1.26.0,<1.27.0)", "mypy-boto3-rds (>=1.26.0,<1.27.0)", "mypy-boto3-rds-data (>=1.26.0,<1.27.0)", "mypy-boto3-redshift (>=1.26.0,<1.27.0)", "mypy-boto3-redshift-data (>=1.26.0,<1.27.0)", "mypy-boto3-redshift-serverless (>=1.26.0,<1.27.0)", "mypy-boto3-rekognition (>=1.26.0,<1.27.0)", "mypy-boto3-resiliencehub (>=1.26.0,<1.27.0)", "mypy-boto3-resource-explorer-2 (>=1.26.0,<1.27.0)", "mypy-boto3-resource-groups (>=1.26.0,<1.27.0)", "mypy-boto3-resourcegroupstaggingapi (>=1.26.0,<1.27.0)", "mypy-boto3-robomaker (>=1.26.0,<1.27.0)", "mypy-boto3-rolesanywhere (>=1.26.0,<1.27.0)", "mypy-boto3-route53 (>=1.26.0,<1.27.0)", "mypy-boto3-route53-recovery-cluster (>=1.26.0,<1.27.0)", "mypy-boto3-route53-recovery-control-config (>=1.26.0,<1.27.0)", "mypy-boto3-route53-recovery-readiness (>=1.26.0,<1.27.0)", "mypy-boto3-route53domains (>=1.26.0,<1.27.0)", "mypy-boto3-route53resolver (>=1.26.0,<1.27.0)", "mypy-boto3-rum (>=1.26.0,<1.27.0)", "mypy-boto3-s3 (>=1.26.0,<1.27.0)", "mypy-boto3-s3control (>=1.26.0,<1.27.0)", "mypy-boto3-s3outposts (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-a2i-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-edge (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-featurestore-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-geospatial (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-metrics (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-savingsplans (>=1.26.0,<1.27.0)", "mypy-boto3-scheduler (>=1.26.0,<1.27.0)", "mypy-boto3-schemas (>=1.26.0,<1.27.0)", "mypy-boto3-sdb (>=1.26.0,<1.27.0)", "mypy-boto3-secretsmanager (>=1.26.0,<1.27.0)", "mypy-boto3-securityhub (>=1.26.0,<1.27.0)", "mypy-boto3-securitylake (>=1.26.0,<1.27.0)", "mypy-boto3-serverlessrepo (>=1.26.0,<1.27.0)", "mypy-boto3-service-quotas (>=1.26.0,<1.27.0)", "mypy-boto3-servicecatalog (>=1.26.0,<1.27.0)", "mypy-boto3-servicecatalog-appregistry (>=1.26.0,<1.27.0)", "mypy-boto3-servicediscovery (>=1.26.0,<1.27.0)", "mypy-boto3-ses (>=1.26.0,<1.27.0)", "mypy-boto3-sesv2 (>=1.26.0,<1.27.0)", "mypy-boto3-shield (>=1.26.0,<1.27.0)", "mypy-boto3-signer (>=1.26.0,<1.27.0)", "mypy-boto3-simspaceweaver (>=1.26.0,<1.27.0)", "mypy-boto3-sms (>=1.26.0,<1.27.0)", "mypy-boto3-sms-voice (>=1.26.0,<1.27.0)", "mypy-boto3-snow-device-management (>=1.26.0,<1.27.0)", "mypy-boto3-snowball (>=1.26.0,<1.27.0)", "mypy-boto3-sns (>=1.26.0,<1.27.0)", "mypy-boto3-sqs (>=1.26.0,<1.27.0)", "mypy-boto3-ssm (>=1.26.0,<1.27.0)", "mypy-boto3-ssm-contacts (>=1.26.0,<1.27.0)", "mypy-boto3-ssm-incidents (>=1.26.0,<1.27.0)", "mypy-boto3-ssm-sap (>=1.26.0,<1.27.0)", "mypy-boto3-sso (>=1.26.0,<1.27.0)", "mypy-boto3-sso-admin (>=1.26.0,<1.27.0)", "mypy-boto3-sso-oidc (>=1.26.0,<1.27.0)", "mypy-boto3-stepfunctions (>=1.26.0,<1.27.0)", "mypy-boto3-storagegateway (>=1.26.0,<1.27.0)", "mypy-boto3-sts (>=1.26.0,<1.27.0)", "mypy-boto3-support (>=1.26.0,<1.27.0)", "mypy-boto3-support-app (>=1.26.0,<1.27.0)", "mypy-boto3-swf (>=1.26.0,<1.27.0)", "mypy-boto3-synthetics (>=1.26.0,<1.27.0)", "mypy-boto3-textract (>=1.26.0,<1.27.0)", "mypy-boto3-timestream-query (>=1.26.0,<1.27.0)", "mypy-boto3-timestream-write (>=1.26.0,<1.27.0)", "mypy-boto3-tnb (>=1.26.0,<1.27.0)", "mypy-boto3-transcribe (>=1.26.0,<1.27.0)", "mypy-boto3-transfer (>=1.26.0,<1.27.0)", "mypy-boto3-translate (>=1.26.0,<1.27.0)", "mypy-boto3-voice-id (>=1.26.0,<1.27.0)", "mypy-boto3-vpc-lattice (>=1.26.0,<1.27.0)", "mypy-boto3-waf (>=1.26.0,<1.27.0)", "mypy-boto3-waf-regional (>=1.26.0,<1.27.0)", "mypy-boto3-wafv2 (>=1.26.0,<1.27.0)", "mypy-boto3-wellarchitected (>=1.26.0,<1.27.0)", "mypy-boto3-wisdom (>=1.26.0,<1.27.0)", "mypy-boto3-workdocs (>=1.26.0,<1.27.0)", "mypy-boto3-worklink (>=1.26.0,<1.27.0)", "mypy-boto3-workmail (>=1.26.0,<1.27.0)", "mypy-boto3-workmailmessageflow (>=1.26.0,<1.27.0)", "mypy-boto3-workspaces (>=1.26.0,<1.27.0)", "mypy-boto3-workspaces-web (>=1.26.0,<1.27.0)", "mypy-boto3-xray (>=1.26.0,<1.27.0)"] +all = ["mypy-boto3-accessanalyzer (>=1.26.0,<1.27.0)", "mypy-boto3-account (>=1.26.0,<1.27.0)", "mypy-boto3-acm (>=1.26.0,<1.27.0)", "mypy-boto3-acm-pca (>=1.26.0,<1.27.0)", "mypy-boto3-alexaforbusiness (>=1.26.0,<1.27.0)", "mypy-boto3-amp (>=1.26.0,<1.27.0)", "mypy-boto3-amplify (>=1.26.0,<1.27.0)", "mypy-boto3-amplifybackend (>=1.26.0,<1.27.0)", "mypy-boto3-amplifyuibuilder (>=1.26.0,<1.27.0)", "mypy-boto3-apigateway (>=1.26.0,<1.27.0)", "mypy-boto3-apigatewaymanagementapi (>=1.26.0,<1.27.0)", "mypy-boto3-apigatewayv2 (>=1.26.0,<1.27.0)", "mypy-boto3-appconfig (>=1.26.0,<1.27.0)", "mypy-boto3-appconfigdata (>=1.26.0,<1.27.0)", "mypy-boto3-appflow (>=1.26.0,<1.27.0)", "mypy-boto3-appintegrations (>=1.26.0,<1.27.0)", "mypy-boto3-application-autoscaling (>=1.26.0,<1.27.0)", "mypy-boto3-application-insights (>=1.26.0,<1.27.0)", "mypy-boto3-applicationcostprofiler (>=1.26.0,<1.27.0)", "mypy-boto3-appmesh (>=1.26.0,<1.27.0)", "mypy-boto3-apprunner (>=1.26.0,<1.27.0)", "mypy-boto3-appstream (>=1.26.0,<1.27.0)", "mypy-boto3-appsync (>=1.26.0,<1.27.0)", "mypy-boto3-arc-zonal-shift (>=1.26.0,<1.27.0)", "mypy-boto3-athena (>=1.26.0,<1.27.0)", "mypy-boto3-auditmanager (>=1.26.0,<1.27.0)", "mypy-boto3-autoscaling (>=1.26.0,<1.27.0)", "mypy-boto3-autoscaling-plans (>=1.26.0,<1.27.0)", "mypy-boto3-backup (>=1.26.0,<1.27.0)", "mypy-boto3-backup-gateway (>=1.26.0,<1.27.0)", "mypy-boto3-backupstorage (>=1.26.0,<1.27.0)", "mypy-boto3-batch (>=1.26.0,<1.27.0)", "mypy-boto3-billingconductor (>=1.26.0,<1.27.0)", "mypy-boto3-braket (>=1.26.0,<1.27.0)", "mypy-boto3-budgets (>=1.26.0,<1.27.0)", "mypy-boto3-ce (>=1.26.0,<1.27.0)", "mypy-boto3-chime (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-identity (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-media-pipelines (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-meetings (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-messaging (>=1.26.0,<1.27.0)", "mypy-boto3-chime-sdk-voice (>=1.26.0,<1.27.0)", "mypy-boto3-cleanrooms (>=1.26.0,<1.27.0)", "mypy-boto3-cloud9 (>=1.26.0,<1.27.0)", "mypy-boto3-cloudcontrol (>=1.26.0,<1.27.0)", "mypy-boto3-clouddirectory (>=1.26.0,<1.27.0)", "mypy-boto3-cloudformation (>=1.26.0,<1.27.0)", "mypy-boto3-cloudfront (>=1.26.0,<1.27.0)", "mypy-boto3-cloudhsm (>=1.26.0,<1.27.0)", "mypy-boto3-cloudhsmv2 (>=1.26.0,<1.27.0)", "mypy-boto3-cloudsearch (>=1.26.0,<1.27.0)", "mypy-boto3-cloudsearchdomain (>=1.26.0,<1.27.0)", "mypy-boto3-cloudtrail (>=1.26.0,<1.27.0)", "mypy-boto3-cloudtrail-data (>=1.26.0,<1.27.0)", "mypy-boto3-cloudwatch (>=1.26.0,<1.27.0)", "mypy-boto3-codeartifact (>=1.26.0,<1.27.0)", "mypy-boto3-codebuild (>=1.26.0,<1.27.0)", "mypy-boto3-codecatalyst (>=1.26.0,<1.27.0)", "mypy-boto3-codecommit (>=1.26.0,<1.27.0)", "mypy-boto3-codedeploy (>=1.26.0,<1.27.0)", "mypy-boto3-codeguru-reviewer (>=1.26.0,<1.27.0)", "mypy-boto3-codeguruprofiler (>=1.26.0,<1.27.0)", "mypy-boto3-codepipeline (>=1.26.0,<1.27.0)", "mypy-boto3-codestar (>=1.26.0,<1.27.0)", "mypy-boto3-codestar-connections (>=1.26.0,<1.27.0)", "mypy-boto3-codestar-notifications (>=1.26.0,<1.27.0)", "mypy-boto3-cognito-identity (>=1.26.0,<1.27.0)", "mypy-boto3-cognito-idp (>=1.26.0,<1.27.0)", "mypy-boto3-cognito-sync (>=1.26.0,<1.27.0)", "mypy-boto3-comprehend (>=1.26.0,<1.27.0)", "mypy-boto3-comprehendmedical (>=1.26.0,<1.27.0)", "mypy-boto3-compute-optimizer (>=1.26.0,<1.27.0)", "mypy-boto3-config (>=1.26.0,<1.27.0)", "mypy-boto3-connect (>=1.26.0,<1.27.0)", "mypy-boto3-connect-contact-lens (>=1.26.0,<1.27.0)", "mypy-boto3-connectcampaigns (>=1.26.0,<1.27.0)", "mypy-boto3-connectcases (>=1.26.0,<1.27.0)", "mypy-boto3-connectparticipant (>=1.26.0,<1.27.0)", "mypy-boto3-controltower (>=1.26.0,<1.27.0)", "mypy-boto3-cur (>=1.26.0,<1.27.0)", "mypy-boto3-customer-profiles (>=1.26.0,<1.27.0)", "mypy-boto3-databrew (>=1.26.0,<1.27.0)", "mypy-boto3-dataexchange (>=1.26.0,<1.27.0)", "mypy-boto3-datapipeline (>=1.26.0,<1.27.0)", "mypy-boto3-datasync (>=1.26.0,<1.27.0)", "mypy-boto3-dax (>=1.26.0,<1.27.0)", "mypy-boto3-detective (>=1.26.0,<1.27.0)", "mypy-boto3-devicefarm (>=1.26.0,<1.27.0)", "mypy-boto3-devops-guru (>=1.26.0,<1.27.0)", "mypy-boto3-directconnect (>=1.26.0,<1.27.0)", "mypy-boto3-discovery (>=1.26.0,<1.27.0)", "mypy-boto3-dlm (>=1.26.0,<1.27.0)", "mypy-boto3-dms (>=1.26.0,<1.27.0)", "mypy-boto3-docdb (>=1.26.0,<1.27.0)", "mypy-boto3-docdb-elastic (>=1.26.0,<1.27.0)", "mypy-boto3-drs (>=1.26.0,<1.27.0)", "mypy-boto3-ds (>=1.26.0,<1.27.0)", "mypy-boto3-dynamodb (>=1.26.0,<1.27.0)", "mypy-boto3-dynamodbstreams (>=1.26.0,<1.27.0)", "mypy-boto3-ebs (>=1.26.0,<1.27.0)", "mypy-boto3-ec2 (>=1.26.0,<1.27.0)", "mypy-boto3-ec2-instance-connect (>=1.26.0,<1.27.0)", "mypy-boto3-ecr (>=1.26.0,<1.27.0)", "mypy-boto3-ecr-public (>=1.26.0,<1.27.0)", "mypy-boto3-ecs (>=1.26.0,<1.27.0)", "mypy-boto3-efs (>=1.26.0,<1.27.0)", "mypy-boto3-eks (>=1.26.0,<1.27.0)", "mypy-boto3-elastic-inference (>=1.26.0,<1.27.0)", "mypy-boto3-elasticache (>=1.26.0,<1.27.0)", "mypy-boto3-elasticbeanstalk (>=1.26.0,<1.27.0)", "mypy-boto3-elastictranscoder (>=1.26.0,<1.27.0)", "mypy-boto3-elb (>=1.26.0,<1.27.0)", "mypy-boto3-elbv2 (>=1.26.0,<1.27.0)", "mypy-boto3-emr (>=1.26.0,<1.27.0)", "mypy-boto3-emr-containers (>=1.26.0,<1.27.0)", "mypy-boto3-emr-serverless (>=1.26.0,<1.27.0)", "mypy-boto3-es (>=1.26.0,<1.27.0)", "mypy-boto3-events (>=1.26.0,<1.27.0)", "mypy-boto3-evidently (>=1.26.0,<1.27.0)", "mypy-boto3-finspace (>=1.26.0,<1.27.0)", "mypy-boto3-finspace-data (>=1.26.0,<1.27.0)", "mypy-boto3-firehose (>=1.26.0,<1.27.0)", "mypy-boto3-fis (>=1.26.0,<1.27.0)", "mypy-boto3-fms (>=1.26.0,<1.27.0)", "mypy-boto3-forecast (>=1.26.0,<1.27.0)", "mypy-boto3-forecastquery (>=1.26.0,<1.27.0)", "mypy-boto3-frauddetector (>=1.26.0,<1.27.0)", "mypy-boto3-fsx (>=1.26.0,<1.27.0)", "mypy-boto3-gamelift (>=1.26.0,<1.27.0)", "mypy-boto3-gamesparks (>=1.26.0,<1.27.0)", "mypy-boto3-glacier (>=1.26.0,<1.27.0)", "mypy-boto3-globalaccelerator (>=1.26.0,<1.27.0)", "mypy-boto3-glue (>=1.26.0,<1.27.0)", "mypy-boto3-grafana (>=1.26.0,<1.27.0)", "mypy-boto3-greengrass (>=1.26.0,<1.27.0)", "mypy-boto3-greengrassv2 (>=1.26.0,<1.27.0)", "mypy-boto3-groundstation (>=1.26.0,<1.27.0)", "mypy-boto3-guardduty (>=1.26.0,<1.27.0)", "mypy-boto3-health (>=1.26.0,<1.27.0)", "mypy-boto3-healthlake (>=1.26.0,<1.27.0)", "mypy-boto3-honeycode (>=1.26.0,<1.27.0)", "mypy-boto3-iam (>=1.26.0,<1.27.0)", "mypy-boto3-identitystore (>=1.26.0,<1.27.0)", "mypy-boto3-imagebuilder (>=1.26.0,<1.27.0)", "mypy-boto3-importexport (>=1.26.0,<1.27.0)", "mypy-boto3-inspector (>=1.26.0,<1.27.0)", "mypy-boto3-inspector2 (>=1.26.0,<1.27.0)", "mypy-boto3-internetmonitor (>=1.26.0,<1.27.0)", "mypy-boto3-iot (>=1.26.0,<1.27.0)", "mypy-boto3-iot-data (>=1.26.0,<1.27.0)", "mypy-boto3-iot-jobs-data (>=1.26.0,<1.27.0)", "mypy-boto3-iot-roborunner (>=1.26.0,<1.27.0)", "mypy-boto3-iot1click-devices (>=1.26.0,<1.27.0)", "mypy-boto3-iot1click-projects (>=1.26.0,<1.27.0)", "mypy-boto3-iotanalytics (>=1.26.0,<1.27.0)", "mypy-boto3-iotdeviceadvisor (>=1.26.0,<1.27.0)", "mypy-boto3-iotevents (>=1.26.0,<1.27.0)", "mypy-boto3-iotevents-data (>=1.26.0,<1.27.0)", "mypy-boto3-iotfleethub (>=1.26.0,<1.27.0)", "mypy-boto3-iotfleetwise (>=1.26.0,<1.27.0)", "mypy-boto3-iotsecuretunneling (>=1.26.0,<1.27.0)", "mypy-boto3-iotsitewise (>=1.26.0,<1.27.0)", "mypy-boto3-iotthingsgraph (>=1.26.0,<1.27.0)", "mypy-boto3-iottwinmaker (>=1.26.0,<1.27.0)", "mypy-boto3-iotwireless (>=1.26.0,<1.27.0)", "mypy-boto3-ivs (>=1.26.0,<1.27.0)", "mypy-boto3-ivs-realtime (>=1.26.0,<1.27.0)", "mypy-boto3-ivschat (>=1.26.0,<1.27.0)", "mypy-boto3-kafka (>=1.26.0,<1.27.0)", "mypy-boto3-kafkaconnect (>=1.26.0,<1.27.0)", "mypy-boto3-kendra (>=1.26.0,<1.27.0)", "mypy-boto3-kendra-ranking (>=1.26.0,<1.27.0)", "mypy-boto3-keyspaces (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-archived-media (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-media (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-signaling (>=1.26.0,<1.27.0)", "mypy-boto3-kinesis-video-webrtc-storage (>=1.26.0,<1.27.0)", "mypy-boto3-kinesisanalytics (>=1.26.0,<1.27.0)", "mypy-boto3-kinesisanalyticsv2 (>=1.26.0,<1.27.0)", "mypy-boto3-kinesisvideo (>=1.26.0,<1.27.0)", "mypy-boto3-kms (>=1.26.0,<1.27.0)", "mypy-boto3-lakeformation (>=1.26.0,<1.27.0)", "mypy-boto3-lambda (>=1.26.0,<1.27.0)", "mypy-boto3-lex-models (>=1.26.0,<1.27.0)", "mypy-boto3-lex-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-lexv2-models (>=1.26.0,<1.27.0)", "mypy-boto3-lexv2-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-license-manager (>=1.26.0,<1.27.0)", "mypy-boto3-license-manager-linux-subscriptions (>=1.26.0,<1.27.0)", "mypy-boto3-license-manager-user-subscriptions (>=1.26.0,<1.27.0)", "mypy-boto3-lightsail (>=1.26.0,<1.27.0)", "mypy-boto3-location (>=1.26.0,<1.27.0)", "mypy-boto3-logs (>=1.26.0,<1.27.0)", "mypy-boto3-lookoutequipment (>=1.26.0,<1.27.0)", "mypy-boto3-lookoutmetrics (>=1.26.0,<1.27.0)", "mypy-boto3-lookoutvision (>=1.26.0,<1.27.0)", "mypy-boto3-m2 (>=1.26.0,<1.27.0)", "mypy-boto3-machinelearning (>=1.26.0,<1.27.0)", "mypy-boto3-macie (>=1.26.0,<1.27.0)", "mypy-boto3-macie2 (>=1.26.0,<1.27.0)", "mypy-boto3-managedblockchain (>=1.26.0,<1.27.0)", "mypy-boto3-marketplace-catalog (>=1.26.0,<1.27.0)", "mypy-boto3-marketplace-entitlement (>=1.26.0,<1.27.0)", "mypy-boto3-marketplacecommerceanalytics (>=1.26.0,<1.27.0)", "mypy-boto3-mediaconnect (>=1.26.0,<1.27.0)", "mypy-boto3-mediaconvert (>=1.26.0,<1.27.0)", "mypy-boto3-medialive (>=1.26.0,<1.27.0)", "mypy-boto3-mediapackage (>=1.26.0,<1.27.0)", "mypy-boto3-mediapackage-vod (>=1.26.0,<1.27.0)", "mypy-boto3-mediastore (>=1.26.0,<1.27.0)", "mypy-boto3-mediastore-data (>=1.26.0,<1.27.0)", "mypy-boto3-mediatailor (>=1.26.0,<1.27.0)", "mypy-boto3-memorydb (>=1.26.0,<1.27.0)", "mypy-boto3-meteringmarketplace (>=1.26.0,<1.27.0)", "mypy-boto3-mgh (>=1.26.0,<1.27.0)", "mypy-boto3-mgn (>=1.26.0,<1.27.0)", "mypy-boto3-migration-hub-refactor-spaces (>=1.26.0,<1.27.0)", "mypy-boto3-migrationhub-config (>=1.26.0,<1.27.0)", "mypy-boto3-migrationhuborchestrator (>=1.26.0,<1.27.0)", "mypy-boto3-migrationhubstrategy (>=1.26.0,<1.27.0)", "mypy-boto3-mobile (>=1.26.0,<1.27.0)", "mypy-boto3-mq (>=1.26.0,<1.27.0)", "mypy-boto3-mturk (>=1.26.0,<1.27.0)", "mypy-boto3-mwaa (>=1.26.0,<1.27.0)", "mypy-boto3-neptune (>=1.26.0,<1.27.0)", "mypy-boto3-network-firewall (>=1.26.0,<1.27.0)", "mypy-boto3-networkmanager (>=1.26.0,<1.27.0)", "mypy-boto3-nimble (>=1.26.0,<1.27.0)", "mypy-boto3-oam (>=1.26.0,<1.27.0)", "mypy-boto3-omics (>=1.26.0,<1.27.0)", "mypy-boto3-opensearch (>=1.26.0,<1.27.0)", "mypy-boto3-opensearchserverless (>=1.26.0,<1.27.0)", "mypy-boto3-opsworks (>=1.26.0,<1.27.0)", "mypy-boto3-opsworkscm (>=1.26.0,<1.27.0)", "mypy-boto3-organizations (>=1.26.0,<1.27.0)", "mypy-boto3-osis (>=1.26.0,<1.27.0)", "mypy-boto3-outposts (>=1.26.0,<1.27.0)", "mypy-boto3-panorama (>=1.26.0,<1.27.0)", "mypy-boto3-personalize (>=1.26.0,<1.27.0)", "mypy-boto3-personalize-events (>=1.26.0,<1.27.0)", "mypy-boto3-personalize-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-pi (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint-email (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint-sms-voice (>=1.26.0,<1.27.0)", "mypy-boto3-pinpoint-sms-voice-v2 (>=1.26.0,<1.27.0)", "mypy-boto3-pipes (>=1.26.0,<1.27.0)", "mypy-boto3-polly (>=1.26.0,<1.27.0)", "mypy-boto3-pricing (>=1.26.0,<1.27.0)", "mypy-boto3-privatenetworks (>=1.26.0,<1.27.0)", "mypy-boto3-proton (>=1.26.0,<1.27.0)", "mypy-boto3-qldb (>=1.26.0,<1.27.0)", "mypy-boto3-qldb-session (>=1.26.0,<1.27.0)", "mypy-boto3-quicksight (>=1.26.0,<1.27.0)", "mypy-boto3-ram (>=1.26.0,<1.27.0)", "mypy-boto3-rbin (>=1.26.0,<1.27.0)", "mypy-boto3-rds (>=1.26.0,<1.27.0)", "mypy-boto3-rds-data (>=1.26.0,<1.27.0)", "mypy-boto3-redshift (>=1.26.0,<1.27.0)", "mypy-boto3-redshift-data (>=1.26.0,<1.27.0)", "mypy-boto3-redshift-serverless (>=1.26.0,<1.27.0)", "mypy-boto3-rekognition (>=1.26.0,<1.27.0)", "mypy-boto3-resiliencehub (>=1.26.0,<1.27.0)", "mypy-boto3-resource-explorer-2 (>=1.26.0,<1.27.0)", "mypy-boto3-resource-groups (>=1.26.0,<1.27.0)", "mypy-boto3-resourcegroupstaggingapi (>=1.26.0,<1.27.0)", "mypy-boto3-robomaker (>=1.26.0,<1.27.0)", "mypy-boto3-rolesanywhere (>=1.26.0,<1.27.0)", "mypy-boto3-route53 (>=1.26.0,<1.27.0)", "mypy-boto3-route53-recovery-cluster (>=1.26.0,<1.27.0)", "mypy-boto3-route53-recovery-control-config (>=1.26.0,<1.27.0)", "mypy-boto3-route53-recovery-readiness (>=1.26.0,<1.27.0)", "mypy-boto3-route53domains (>=1.26.0,<1.27.0)", "mypy-boto3-route53resolver (>=1.26.0,<1.27.0)", "mypy-boto3-rum (>=1.26.0,<1.27.0)", "mypy-boto3-s3 (>=1.26.0,<1.27.0)", "mypy-boto3-s3control (>=1.26.0,<1.27.0)", "mypy-boto3-s3outposts (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-a2i-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-edge (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-featurestore-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-geospatial (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-metrics (>=1.26.0,<1.27.0)", "mypy-boto3-sagemaker-runtime (>=1.26.0,<1.27.0)", "mypy-boto3-savingsplans (>=1.26.0,<1.27.0)", "mypy-boto3-scheduler (>=1.26.0,<1.27.0)", "mypy-boto3-schemas (>=1.26.0,<1.27.0)", "mypy-boto3-sdb (>=1.26.0,<1.27.0)", "mypy-boto3-secretsmanager (>=1.26.0,<1.27.0)", "mypy-boto3-securityhub (>=1.26.0,<1.27.0)", "mypy-boto3-securitylake (>=1.26.0,<1.27.0)", "mypy-boto3-serverlessrepo (>=1.26.0,<1.27.0)", "mypy-boto3-service-quotas (>=1.26.0,<1.27.0)", "mypy-boto3-servicecatalog (>=1.26.0,<1.27.0)", "mypy-boto3-servicecatalog-appregistry (>=1.26.0,<1.27.0)", "mypy-boto3-servicediscovery (>=1.26.0,<1.27.0)", "mypy-boto3-ses (>=1.26.0,<1.27.0)", "mypy-boto3-sesv2 (>=1.26.0,<1.27.0)", "mypy-boto3-shield (>=1.26.0,<1.27.0)", "mypy-boto3-signer (>=1.26.0,<1.27.0)", "mypy-boto3-simspaceweaver (>=1.26.0,<1.27.0)", "mypy-boto3-sms (>=1.26.0,<1.27.0)", "mypy-boto3-sms-voice (>=1.26.0,<1.27.0)", "mypy-boto3-snow-device-management (>=1.26.0,<1.27.0)", "mypy-boto3-snowball (>=1.26.0,<1.27.0)", "mypy-boto3-sns (>=1.26.0,<1.27.0)", "mypy-boto3-sqs (>=1.26.0,<1.27.0)", "mypy-boto3-ssm (>=1.26.0,<1.27.0)", "mypy-boto3-ssm-contacts (>=1.26.0,<1.27.0)", "mypy-boto3-ssm-incidents (>=1.26.0,<1.27.0)", "mypy-boto3-ssm-sap (>=1.26.0,<1.27.0)", "mypy-boto3-sso (>=1.26.0,<1.27.0)", "mypy-boto3-sso-admin (>=1.26.0,<1.27.0)", "mypy-boto3-sso-oidc (>=1.26.0,<1.27.0)", "mypy-boto3-stepfunctions (>=1.26.0,<1.27.0)", "mypy-boto3-storagegateway (>=1.26.0,<1.27.0)", "mypy-boto3-sts (>=1.26.0,<1.27.0)", "mypy-boto3-support (>=1.26.0,<1.27.0)", "mypy-boto3-support-app (>=1.26.0,<1.27.0)", "mypy-boto3-swf (>=1.26.0,<1.27.0)", "mypy-boto3-synthetics (>=1.26.0,<1.27.0)", "mypy-boto3-textract (>=1.26.0,<1.27.0)", "mypy-boto3-timestream-query (>=1.26.0,<1.27.0)", "mypy-boto3-timestream-write (>=1.26.0,<1.27.0)", "mypy-boto3-tnb (>=1.26.0,<1.27.0)", "mypy-boto3-transcribe (>=1.26.0,<1.27.0)", "mypy-boto3-transfer (>=1.26.0,<1.27.0)", "mypy-boto3-translate (>=1.26.0,<1.27.0)", "mypy-boto3-voice-id (>=1.26.0,<1.27.0)", "mypy-boto3-vpc-lattice (>=1.26.0,<1.27.0)", "mypy-boto3-waf (>=1.26.0,<1.27.0)", "mypy-boto3-waf-regional (>=1.26.0,<1.27.0)", "mypy-boto3-wafv2 (>=1.26.0,<1.27.0)", "mypy-boto3-wellarchitected (>=1.26.0,<1.27.0)", "mypy-boto3-wisdom (>=1.26.0,<1.27.0)", "mypy-boto3-workdocs (>=1.26.0,<1.27.0)", "mypy-boto3-worklink (>=1.26.0,<1.27.0)", "mypy-boto3-workmail (>=1.26.0,<1.27.0)", "mypy-boto3-workmailmessageflow (>=1.26.0,<1.27.0)", "mypy-boto3-workspaces (>=1.26.0,<1.27.0)", "mypy-boto3-workspaces-web (>=1.26.0,<1.27.0)", "mypy-boto3-xray (>=1.26.0,<1.27.0)"] amp = ["mypy-boto3-amp (>=1.26.0,<1.27.0)"] amplify = ["mypy-boto3-amplify (>=1.26.0,<1.27.0)"] amplifybackend = ["mypy-boto3-amplifybackend (>=1.26.0,<1.27.0)"] @@ -235,7 +247,7 @@ backup-gateway = ["mypy-boto3-backup-gateway (>=1.26.0,<1.27.0)"] backupstorage = ["mypy-boto3-backupstorage (>=1.26.0,<1.27.0)"] batch = ["mypy-boto3-batch (>=1.26.0,<1.27.0)"] billingconductor = ["mypy-boto3-billingconductor (>=1.26.0,<1.27.0)"] -boto3 = ["boto3 (==1.26.117)", "botocore (==1.29.117)"] +boto3 = ["boto3 (==1.26.135)", "botocore (==1.29.135)"] braket = ["mypy-boto3-braket (>=1.26.0,<1.27.0)"] budgets = ["mypy-boto3-budgets (>=1.26.0,<1.27.0)"] ce = ["mypy-boto3-ce (>=1.26.0,<1.27.0)"] @@ -440,6 +452,7 @@ opensearchserverless = ["mypy-boto3-opensearchserverless (>=1.26.0,<1.27.0)"] opsworks = ["mypy-boto3-opsworks (>=1.26.0,<1.27.0)"] opsworkscm = ["mypy-boto3-opsworkscm (>=1.26.0,<1.27.0)"] organizations = ["mypy-boto3-organizations (>=1.26.0,<1.27.0)"] +osis = ["mypy-boto3-osis (>=1.26.0,<1.27.0)"] outposts = ["mypy-boto3-outposts (>=1.26.0,<1.27.0)"] panorama = ["mypy-boto3-panorama (>=1.26.0,<1.27.0)"] personalize = ["mypy-boto3-personalize (>=1.26.0,<1.27.0)"] @@ -550,14 +563,14 @@ xray = ["mypy-boto3-xray (>=1.26.0,<1.27.0)"] [[package]] name = "botocore" -version = "1.29.117" +version = "1.29.135" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.29.117-py3-none-any.whl", hash = "sha256:2e6f6c9be770ed5737de093cfe8d592be15f8763f7bf2906afba90f19836d72d"}, - {file = "botocore-1.29.117.tar.gz", hash = "sha256:1fc4469e80daab657d1254518ee44a82e284320c8b24cbfbaaa60570afc33743"}, + {file = "botocore-1.29.135-py3-none-any.whl", hash = "sha256:06502a4473924ef60ac0de908385a5afab9caee6c5b49cf6a330fab0d76ddf5f"}, + {file = "botocore-1.29.135.tar.gz", hash = "sha256:0c61d4e5e04fe5329fa65da6b31492ef9d0d5174d72fc2af69de2ed0f87804ca"}, ] [package.dependencies] @@ -570,14 +583,14 @@ crt = ["awscrt (==0.16.9)"] [[package]] name = "botocore-stubs" -version = "1.29.117" +version = "1.29.130" description = "Type annotations and code completion for botocore" category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "botocore_stubs-1.29.117-py3-none-any.whl", hash = "sha256:80e494f6d6dfe9f141900176c0857988f6c333030348b7590b4ec515a6971cf1"}, - {file = "botocore_stubs-1.29.117.tar.gz", hash = "sha256:f4de553d1106624884ce198fe9d2ea030bfee7a78e395df827a8c79d04e9f33a"}, + {file = "botocore_stubs-1.29.130-py3-none-any.whl", hash = "sha256:622c4a5cd740498439008d81c5ded612146f4f0d575341c12591f978edbbe733"}, + {file = "botocore_stubs-1.29.130.tar.gz", hash = "sha256:5f6f1967d23c45834858a055cbf65b66863f9f28d05f32f57bf52864a13512d9"}, ] [package.dependencies] @@ -586,14 +599,14 @@ typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.9\""} [[package]] name = "certifi" -version = "2022.12.7" +version = "2023.5.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, - {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, + {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, + {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, ] [[package]] @@ -1040,21 +1053,21 @@ setuptools = "*" [[package]] name = "dcicsnovault" -version = "7.3.0" +version = "8.0.2b0" description = "Storage support for 4DN Data Portals." category = "main" optional = false python-versions = ">=3.8.1,<3.9" files = [ - {file = "dcicsnovault-7.3.0-py3-none-any.whl", hash = "sha256:79364712ca035fcce07d24815fbec48d812ede1d1c4ed6aed82b9d739a95f4c1"}, - {file = "dcicsnovault-7.3.0.tar.gz", hash = "sha256:12170eaf3bcf0f753335b5987952bd48a6806548a663a98c533ba0ef0da5a19c"}, + {file = "dcicsnovault-8.0.2b0-py3-none-any.whl", hash = "sha256:6e5979025e95d81db8217e8a794e707ccccb022cb77027807b61f41364db47af"}, + {file = "dcicsnovault-8.0.2b0.tar.gz", hash = "sha256:0cf983393e61f964bce358acac5943b5c8db2e5c340e081a3733056776ed5910"}, ] [package.dependencies] aws_requests_auth = ">=0.4.1,<0.5.0" boto3 = ">=1.24.36" botocore = ">=1.27.36" -dcicutils = ">=6.10.0,<7.0.0" +dcicutils = ">=7.0.0,<8.0.0" elasticsearch = "7.13.4" elasticsearch_dsl = ">=7.4.0,<8.0.0" future = ">=0.15.2,<1" @@ -1072,11 +1085,13 @@ pyramid-multiauth = ">=0.9.0,<1" pyramid-retry = ">=1.0,<2.0" pyramid-tm = ">=2.5,<3.0" pyramid-translogger = ">=0.1,<0.2" +pytest-redis = ">=2.0.0,<3.0.0" python-dateutil = ">=2.8.2,<3.0.0" python_magic = ">=0.4.27" pytz = ">=2021.3" rdflib = ">=4.2.2,<5.0.0" rdflib-jsonld = ">=0.5.0,<1.0.0" +redis = ">=4.5.1,<5.0.0" rutter = ">=0.3,<1" simplejson = ">=3.17.6,<4.0.0" SPARQLWrapper = ">=1.8.5,<2.0.0" @@ -1095,14 +1110,14 @@ xlrd = ">=1.0.0,<2.0.0" [[package]] name = "dcicutils" -version = "6.10.1" +version = "7.4.2" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" category = "main" optional = false python-versions = ">=3.7,<3.10" files = [ - {file = "dcicutils-6.10.1-py3-none-any.whl", hash = "sha256:50c380912bd29d420d07f0646a9733d52c2a667e55db9d8bc7d2d2cb0f16f9f9"}, - {file = "dcicutils-6.10.1.tar.gz", hash = "sha256:dcfa5fa38bcc2184a5586034cbd64eb9cc72b93041cdbfd3c6cc1a5bc31a556a"}, + {file = "dcicutils-7.4.2-py3-none-any.whl", hash = "sha256:844031525f9805cc41ed98ef879792889bf75af277e50f97c47cdb203a96ba73"}, + {file = "dcicutils-7.4.2.tar.gz", hash = "sha256:4f119aa5ef34387deca91631df2036ed15dbae34db3a5d72ed4052926608e3d7"}, ] [package.dependencies] @@ -1113,12 +1128,16 @@ docker = ">=4.4.4,<5.0.0" elasticsearch = "7.13.4" gitpython = ">=3.1.2,<4.0.0" opensearch-py = ">=2.0.1,<3.0.0" +PyJWT = ">=2.6.0,<3.0.0" +pyOpenSSL = ">=23.1.1,<24.0.0" pytz = ">=2020.4" PyYAML = ">=5.1,<5.5" +redis = ">=4.5.1,<5.0.0" requests = ">=2.21.0,<3.0.0" rfc3986 = ">=1.4.0,<2.0.0" structlog = ">=19.2.0,<20.0.0" toml = ">=0.10.1,<1" +tqdm = ">=4.65.0,<5.0.0" typing-extensions = ">=3.8" urllib3 = ">=1.26.6,<2.0.0" webtest = ">=2.0.34,<3.0.0" @@ -1226,7 +1245,7 @@ files = [ name = "exceptiongroup" version = "1.1.1" description = "Backport of PEP 654 (exception groups)" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1541,14 +1560,14 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.5.0" +version = "6.6.0" description = "Read metadata from Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "importlib_metadata-6.5.0-py3-none-any.whl", hash = "sha256:03ba783c3a2c69d751b109fc0c94a62c51f581b3d6acf8ed1331b6d5729321ff"}, - {file = "importlib_metadata-6.5.0.tar.gz", hash = "sha256:7a8bdf1bc3a726297f5cfbc999e6e7ff6b4fa41b26bba4afc580448624460045"}, + {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"}, + {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"}, ] [package.dependencies] @@ -1563,7 +1582,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1832,6 +1851,21 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mirakuru" +version = "2.5.1" +description = "Process executor (not only) for tests." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mirakuru-2.5.1-py3-none-any.whl", hash = "sha256:0a16f897841741f8cd784f790e54d74e61456ba36be9cb9de731b49e2e7a45dc"}, + {file = "mirakuru-2.5.1.tar.gz", hash = "sha256:5a60d641fa92c8bfcd383f6e52f7a0bf3f081da0467fc6e3e6a3f6b3e3e47a7b"}, +] + +[package.dependencies] +psutil = {version = ">=4.0.0", markers = "sys_platform != \"cygwin\""} + [[package]] name = "moto" version = "4.0.3" @@ -1906,40 +1940,40 @@ files = [ [[package]] name = "numpy" -version = "1.24.2" +version = "1.24.3" description = "Fundamental package for array computing in Python" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, - {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, - {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, - {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, - {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, - {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, - {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, - {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, - {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, - {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, - {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, - {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, - {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, - {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, - {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, - {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, - {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, - {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, - {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, + {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, + {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, + {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, + {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, + {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, + {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, + {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, + {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, + {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, + {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, + {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, + {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, + {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, + {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, + {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, + {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, + {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, + {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, ] [[package]] @@ -1986,7 +2020,7 @@ kerberos = ["requests-kerberos"] name = "packaging" version = "23.1" description = "Core utilities for Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2138,19 +2172,19 @@ test = ["docutils", "pytest-cov", "pytest-pycodestyle", "pytest-runner"] [[package]] name = "pipdeptree" -version = "2.7.0" +version = "2.7.1" description = "Command line utility to show dependency tree of packages." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pipdeptree-2.7.0-py3-none-any.whl", hash = "sha256:f1ed934abb3f5e561ae22118d93d45132d174b94a3664396a4a3f99494f79028"}, - {file = "pipdeptree-2.7.0.tar.gz", hash = "sha256:1c79e28267ddf90ea2293f982db4f5df7a76befca483c68da6c83c4370989e8d"}, + {file = "pipdeptree-2.7.1-py3-none-any.whl", hash = "sha256:bb0ffa98a49b0b4076364b367d1df37fcf6628ec3b5cbb61cf4bbaedc7502db0"}, + {file = "pipdeptree-2.7.1.tar.gz", hash = "sha256:550bd7679379e7290739384f3e9518835620e814cc29ba709513952b627da506"}, ] [package.extras] graphviz = ["graphviz (>=0.20.1)"] -test = ["covdefaults (>=2.3)", "diff-cover (>=7.5)", "pip (>=23.0.1)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "virtualenv (>=20.21,<21)"] +test = ["covdefaults (>=2.3)", "diff-cover (>=7.5)", "pip (>=23.1)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "virtualenv (>=20.21,<21)"] [[package]] name = "plaster" @@ -2194,7 +2228,7 @@ testing = ["pytest", "pytest-cov"] name = "pluggy" version = "1.0.0" description = "plugin and hook calling mechanisms for python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2206,6 +2240,18 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "port-for" +version = "0.6.3" +description = "Utility that helps with local TCP ports management. It can find an unused TCP localhost port and remember the association." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "port-for-0.6.3.tar.gz", hash = "sha256:232bd999015b7fbdf19f90f3a9298cc742252d67650108123940bfc75c6f4d4e"}, + {file = "port_for-0.6.3-py3-none-any.whl", hash = "sha256:31860afde6cb552e1830c927def3288350c8fbbe9aea8aed8150ed9d1aa0de81"}, +] + [[package]] name = "psutil" version = "5.9.5" @@ -2392,20 +2438,40 @@ files = [ [[package]] name = "pyjwt" -version = "1.5.3" +version = "2.7.0" description = "JSON Web Token implementation in Python" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.7.0-py3-none-any.whl", hash = "sha256:ba2b425b15ad5ef12f200dc67dd56af4e26de2331f965c5439994dad075876e1"}, + {file = "PyJWT-2.7.0.tar.gz", hash = "sha256:bd6ca4a3c4285c1a2d4349e5a035fdf8fb94e04ccd0fcbe6ba289dae9cc3e074"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyopenssl" +version = "23.1.1" +description = "Python wrapper module around the OpenSSL library" +category = "main" +optional = false +python-versions = ">=3.6" files = [ - {file = "PyJWT-1.5.3-py2.py3-none-any.whl", hash = "sha256:a4e5f1441e3ca7b382fd0c0b416777ced1f97c64ef0c33bfa39daf38505cfd2f"}, - {file = "PyJWT-1.5.3.tar.gz", hash = "sha256:500be75b17a63f70072416843dc80c8821109030be824f4d14758f114978bae7"}, + {file = "pyOpenSSL-23.1.1-py3-none-any.whl", hash = "sha256:9e0c526404a210df9d2b18cd33364beadb0dc858a739b885677bc65e105d4a4c"}, + {file = "pyOpenSSL-23.1.1.tar.gz", hash = "sha256:841498b9bec61623b1b6c47ebbc02367c07d60e0e195f19790817f10cc8db0b7"}, ] +[package.dependencies] +cryptography = ">=38.0.0,<41" + [package.extras] -crypto = ["cryptography (>=1.4)"] -flake8 = ["flake8", "flake8-import-order", "pep8-naming"] -test = ["pytest (>3,<4)", "pytest-cov", "pytest-runner"] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] +test = ["flaky", "pretend", "pytest (>=3.0.1)"] [[package]] name = "pyparsing" @@ -2581,7 +2647,7 @@ files = [ name = "pytest" version = "7.3.1" description = "pytest: simple powerful testing with Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2652,6 +2718,27 @@ pytest = ">=5.0" [package.extras] dev = ["pre-commit", "pytest-asyncio", "tox"] +[[package]] +name = "pytest-redis" +version = "2.4.0" +description = "Redis fixtures and fixture factories for Pytest." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-redis-2.4.0.tar.gz", hash = "sha256:8a07520abed3cd341e8da1793059aa5717b02e56c43e7c76435db682cede10aa"}, + {file = "pytest_redis-2.4.0-py3-none-any.whl", hash = "sha256:3cf00ad3f7241e38ce6f1bcb66af11b91956a889f1e216cfc026e81aa638a4e7"}, +] + +[package.dependencies] +mirakuru = "*" +port-for = ">=0.6.0" +pytest = ">=6.2.0" +redis = "*" + +[package.extras] +tests = ["mock", "pytest-cov", "pytest-xdist"] + [[package]] name = "pytest-timeout" version = "2.1.0" @@ -2823,6 +2910,25 @@ files = [ [package.dependencies] rdflib = "*" +[[package]] +name = "redis" +version = "4.5.5" +description = "Python client for Redis database and key-value store" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "redis-4.5.5-py3-none-any.whl", hash = "sha256:77929bc7f5dab9adf3acba2d3bb7d7658f1e0c2f1cafe7eb36434e751c471119"}, + {file = "redis-4.5.5.tar.gz", hash = "sha256:dc87a0bdef6c8bfe1ef1e1c40be7034390c2ae02d92dcd0c7ca1729443899880"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.2", markers = "python_full_version <= \"3.11.2\""} + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + [[package]] name = "repoze-debug" version = "1.1" @@ -2844,21 +2950,21 @@ testing = ["WebOb", "coverage", "nose"] [[package]] name = "requests" -version = "2.28.2" +version = "2.30.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=3.7, <4" +python-versions = ">=3.7" files = [ - {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, - {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, + {file = "requests-2.30.0-py3-none-any.whl", hash = "sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294"}, + {file = "requests-2.30.0.tar.gz", hash = "sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4"}, ] [package.dependencies] certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<1.27" +urllib3 = ">=1.21.1,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] @@ -2935,14 +3041,14 @@ testing = ["WebTest", "coverage", "pytest", "pytest-cov"] [[package]] name = "s3transfer" -version = "0.6.0" +version = "0.6.1" description = "An Amazon S3 Transfer Manager" category = "main" optional = false python-versions = ">= 3.7" files = [ - {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, - {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, + {file = "s3transfer-0.6.1-py3-none-any.whl", hash = "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346"}, + {file = "s3transfer-0.6.1.tar.gz", hash = "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9"}, ] [package.dependencies] @@ -2953,19 +3059,19 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] [[package]] name = "sentry-sdk" -version = "1.20.0" +version = "1.23.1" description = "Python client for Sentry (https://sentry.io)" category = "main" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.20.0.tar.gz", hash = "sha256:a3410381ae769a436c0852cce140a5e5e49f566a07fb7c2ab445af1302f6ad89"}, - {file = "sentry_sdk-1.20.0-py2.py3-none-any.whl", hash = "sha256:0ad6bbbe78057b8031a07de7aca6d2a83234e51adc4d436eaf8d8c697184db71"}, + {file = "sentry-sdk-1.23.1.tar.gz", hash = "sha256:0300fbe7a07b3865b3885929fb863a68ff01f59e3bcfb4e7953d0bf7fd19c67f"}, + {file = "sentry_sdk-1.23.1-py2.py3-none-any.whl", hash = "sha256:a884e2478e0b055776ea2b9234d5de9339b4bae0b3a5e74ae43d131db8ded27e"}, ] [package.dependencies] certifi = "*" -urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} +urllib3 = {version = ">=1.26.11,<2.0.0", markers = "python_version >= \"3.6\""} [package.extras] aiohttp = ["aiohttp (>=3.5)"] @@ -2977,10 +3083,11 @@ chalice = ["chalice (>=1.16.0)"] django = ["django (>=1.8)"] falcon = ["falcon (>=1.4)"] fastapi = ["fastapi (>=0.79.0)"] -flask = ["blinker (>=1.1)", "flask (>=0.11)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)", "markupsafe"] grpcio = ["grpcio (>=1.21.1)"] httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] +loguru = ["loguru (>=0.5)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] pure-eval = ["asttokens", "executing", "pure-eval"] pymongo = ["pymongo (>=3.1)"] @@ -2995,14 +3102,14 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "67.7.1" +version = "67.7.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.7.1-py3-none-any.whl", hash = "sha256:6f0839fbdb7e3cfef1fc38d7954f5c1c26bf4eebb155a55c9bf8faf997b9fb67"}, - {file = "setuptools-67.7.1.tar.gz", hash = "sha256:bb16732e8eb928922eabaa022f881ae2b7cdcfaf9993ef1f5e841a96d32b8e0c"}, + {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, + {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, ] [package.extras] @@ -3307,7 +3414,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3370,14 +3477,14 @@ files = [ [[package]] name = "types-awscrt" -version = "0.16.15" +version = "0.16.17" description = "Type annotations and code completion for awscrt" category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "types_awscrt-0.16.15-py3-none-any.whl", hash = "sha256:89d052f4c018ee0fc5d6ed8ae64c2e1f9f1078a2f9a29f5246a42af72f7f4e77"}, - {file = "types_awscrt-0.16.15.tar.gz", hash = "sha256:9f858e9b6237b58bc550d0fb5785758aba2b4e454a71edf81882b5908b37174c"}, + {file = "types_awscrt-0.16.17-py3-none-any.whl", hash = "sha256:e28fb3f20568ce9e96e33e01e0b87b891822f36b8f368adb582553b016d4aa08"}, + {file = "types_awscrt-0.16.17.tar.gz", hash = "sha256:9e447df3ad46767887d14fa9c856df94f80e8a0a7f0169577ab23b52ee37bcdf"}, ] [[package]] @@ -3394,14 +3501,14 @@ files = [ [[package]] name = "types-s3transfer" -version = "0.6.0.post7" +version = "0.6.1" description = "Type annotations and code completion for s3transfer" category = "dev" optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "types_s3transfer-0.6.0.post7-py3-none-any.whl", hash = "sha256:d9c669b30fdd61347720434aacb8ecc4645d900712a70b10f495104f9039c07b"}, - {file = "types_s3transfer-0.6.0.post7.tar.gz", hash = "sha256:40e665643f0647832d51c4a26d8a8275cda9134b02bf22caf28198b79bcad382"}, + {file = "types_s3transfer-0.6.1-py3-none-any.whl", hash = "sha256:6d1ac1dedac750d570428362acdf60fdd4f277b0788855c3894d3226756b2bfb"}, + {file = "types_s3transfer-0.6.1.tar.gz", hash = "sha256:75ac1d7143d58c1e6af467cfd4a96c67ee058a3adf7c249d9309999e1f5f41e4"}, ] [package.dependencies] @@ -3728,4 +3835,4 @@ test = ["zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.9" -content-hash = "fe837842d2850dca542fc45eebc8d16c59d3eb04961dcd82c2e63ef988d9f963" +content-hash = "e538f176102e70deb97f3fdcc7cc137a76b9adddfee7f752cf8120d998cd639d" diff --git a/pyproject.toml b/pyproject.toml index 79c479de62..20053a37c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,8 @@ codeguru-profiler-agent = "^1.2.4" cryptography = "39.0.2" colorama = "0.3.3" dcicpyvcf = "1.0.0.1b0" # "^1.0.0" -dcicsnovault = "7.3.0" -dcicutils = "^6.10.0" +dcicsnovault = "8.0.2b0" +dcicutils = "^7.0.0" elasticsearch = "7.13.4" execnet = "1.4.1" future = ">=0.18.2,<1" @@ -81,7 +81,7 @@ py = ">=1.10.0" # used by pytest, not sure if elsewhere pyasn1 = "0.1.9" PyBrowserID = "^0.14.0" pycparser = "2.14" -PyJWT = "1.5.3" +PyJWT = "^2.6.0" pyparsing = "^3.0.7" pyramid = "1.10.4" pyramid_localroles = ">=0.1,<1" diff --git a/src/encoded/batch_download.py b/src/encoded/batch_download.py index 2c4dee5ffb..7687b06e1b 100644 --- a/src/encoded/batch_download.py +++ b/src/encoded/batch_download.py @@ -1,130 +1,177 @@ -import datetime -import json +from dataclasses import dataclass +from datetime import datetime +from functools import partial +from typing import ( + Callable, + Iterable, + Iterator, + List, + Optional, +) + import pytz import structlog - -from pyramid.httpexceptions import HTTPBadRequest # , HTTPMovedPermanently, HTTPServerError, HTTPTemporaryRedirect, -from pyramid.view import view_config +from pyramid.httpexceptions import HTTPBadRequest +from pyramid.request import Request from pyramid.response import Response -# from pyramid.traversal import find_resource -from snovault.embed import make_subrequest -from snovault.util import simple_path_ids, debug_log +from pyramid.view import view_config +from snovault.util import debug_log, simple_path_ids -from .batch_download_utils import stream_tsv_output, convert_item_to_sheet_dict, human_readable_filter_block_queries -from .search.compound_search import CompoundSearchBuilder -from .types.variant import get_spreadsheet_mappings +from .item_models import Note, VariantSample +from .batch_download_utils import ( + ACCEPTABLE_FILE_FORMATS, + get_values_for_field, + human_readable_filter_block_queries, + FilterSetSearch, + OrderedSpreadsheetColumn, + SpreadsheetColumn, + SpreadsheetFromColumnTuples, + SpreadsheetCreationError, + SpreadsheetGenerator, + SpreadsheetRequest, +) +from .root import CGAPRoot +from .util import ( + APPLICATION_FORM_ENCODED_MIME_TYPE, + JsonObject, + format_to_url, + register_path_content_type, +) log = structlog.getLogger(__name__) +CASE_SPREADSHEET_ENDPOINT = "case_search_spreadsheet" +CASE_SPREADSHEET_URL = format_to_url(CASE_SPREADSHEET_ENDPOINT) +VARIANT_SAMPLE_SPREADSHEET_ENDPOINT = "variant_sample_search_spreadsheet" +VARIANT_SAMPLE_SPREADSHEET_URL = format_to_url(VARIANT_SAMPLE_SPREADSHEET_ENDPOINT) + + def includeme(config): - config.add_route('variant_sample_search_spreadsheet', '/variant-sample-search-spreadsheet/') + config.add_route(CASE_SPREADSHEET_ENDPOINT, CASE_SPREADSHEET_URL) + config.add_route( + VARIANT_SAMPLE_SPREADSHEET_ENDPOINT, VARIANT_SAMPLE_SPREADSHEET_URL + ) config.scan(__name__) +register_path_content_type( + path=CASE_SPREADSHEET_URL, content_type=APPLICATION_FORM_ENCODED_MIME_TYPE +) -########################################################################### -## Spreadsheet Generation Code Specific to VariantSample Search Requests ## -########################################################################### +def validate_spreadsheet_file_format(context: CGAPRoot, request: Request) -> None: + """Validate file format is accepted.""" + spreadsheet_request = SpreadsheetRequest(request) + file_format = spreadsheet_request.get_file_format() + if file_format not in ACCEPTABLE_FILE_FORMATS: + raise HTTPBadRequest(f"File format not acceptable: {file_format}") +def validate_spreadsheet_search_parameters(context: CGAPRoot, request: Request) -> None: + """Validate search parameters provided.""" + spreadsheet_request = SpreadsheetRequest(request) + search = spreadsheet_request.get_compound_search() + if not search: + raise HTTPBadRequest("No search parameters given") -@view_config(route_name='variant_sample_search_spreadsheet', request_method=['GET', 'POST']) + +@view_config( + route_name=VARIANT_SAMPLE_SPREADSHEET_ENDPOINT, + request_method="POST", + validators=[ + validate_spreadsheet_file_format, + validate_spreadsheet_search_parameters, + ], +) @debug_log -def variant_sample_search_spreadsheet(context, request): - """ - Returns spreadsheet - """ - - request_body = {} - try: - # This is what we should be receiving - request_body = request.POST - except Exception: - # TODO: Consider accepting JSON body for unit test purposes only - pass - - - file_format = request_body.get("file_format", request.GET.get("file_format", "tsv")).lower() - if file_format not in { "tsv", "csv" }: # TODO: Add support for xslx. - raise HTTPBadRequest("Expected a valid `file_format` such as TSV or CSV.") - - case_accession = request_body.get("case_accession", request.GET.get("case_accession")) - case_title = request_body.get("case_title", request.GET.get("case_title")) - - timestamp = datetime.datetime.now(pytz.utc).isoformat()[:-13] + "Z" - suggested_filename = (case_accession or "case") + "-filtering-" + timestamp + "." + file_format - - spreadsheet_mappings = get_spreadsheet_mappings(request) - - - # Must not contain `limit` - filterset_blocks_request = request_body["compound_search_request"] - if isinstance(filterset_blocks_request, str): - # Assuming is from a www-form-encoded POST request, which has value stringified. - filterset_blocks_request = json.loads(filterset_blocks_request) - - filter_set = CompoundSearchBuilder.extract_filter_set_from_search_body(request, filterset_blocks_request) - global_flags = filterset_blocks_request.get('global_flags', None) - intersect = True if filterset_blocks_request.get('intersect', False) else False - - compound_search_res = CompoundSearchBuilder.execute_filter_set( - context, - request, - filter_set, - from_=0, - to="all", - global_flags=global_flags, - intersect=intersect, - return_generator=True +def variant_sample_search_spreadsheet(context: CGAPRoot, request: Request) -> Response: + """Download spreadsheet for VariantSamples from search.""" + spreadsheet_request = SpreadsheetRequest(request) + file_format = spreadsheet_request.get_file_format() + file_name = get_variant_sample_spreadsheet_file_name(spreadsheet_request) + items_for_spreadsheet = get_items_from_search(context, request, spreadsheet_request) + spreadsheet_rows = get_variant_sample_rows( + items_for_spreadsheet, spreadsheet_request ) + return get_spreadsheet_response(file_name, spreadsheet_rows, file_format) - def vs_dicts_generator(): - for embedded_representation_variant_sample in compound_search_res: - # Extends `embedded_representation_variant_sample` in place - embed_and_merge_note_items_to_variant_sample(request, embedded_representation_variant_sample) - yield convert_item_to_sheet_dict(embedded_representation_variant_sample, spreadsheet_mappings) - - - header_info_rows = [ - ["#"], - ["#", "Case Accession:", "", case_accession or "Not Available"], - ["#", "Case Title:", "", case_title or "Not Available"], - ["#", "Filters Selected:", "", human_readable_filter_block_queries(filterset_blocks_request) ], - #["#", "Filtering Query Used:", "", json.dumps({ "intersect": intersect, "filter_blocks": [ fb["query"] for fb in filter_set["filter_blocks"] ] }) ], - ["#"], - ["## -------------------------------------------------------"] # <- Slightly less than horizontal length of most VS @IDs - ] +@view_config( + route_name=CASE_SPREADSHEET_ENDPOINT, + request_method="POST", + validators=[ + validate_spreadsheet_file_format, + validate_spreadsheet_search_parameters, + ], +) +@debug_log +def case_search_spreadsheet(context: CGAPRoot, request: Request) -> Response: + """Download spreadsheet for Cases from search.""" + spreadsheet_request = SpreadsheetRequest(request) + file_format = spreadsheet_request.get_file_format() + file_name = get_case_spreadsheet_file_name() + items_for_spreadsheet = get_items_from_search(context, request, spreadsheet_request) + spreadsheet_rows = get_case_rows(items_for_spreadsheet) + return get_spreadsheet_response(file_name, spreadsheet_rows, file_format) - return Response( - app_iter = stream_tsv_output( - vs_dicts_generator(), - spreadsheet_mappings, - file_format=file_format, - header_rows=header_info_rows - ), - headers={ - 'X-Accel-Buffering': 'no', - # 'Content-Encoding': 'utf-8', # Commented out -- unit test's TestApp won't decode otherwise. - 'Content-Disposition': 'attachment; filename=' + suggested_filename, - 'Content-Type': 'text/' + file_format, - 'Content-Description': 'File Transfer', - 'Cache-Control': 'no-store' - } - ) + +def get_variant_sample_spreadsheet_file_name( + spreadsheet_request: SpreadsheetRequest, +) -> str: + case_accession = spreadsheet_request.get_case_accession() or "case" + timestamp = get_timestamp() + return f"{case_accession}-filtering-{timestamp}" + + +def get_timestamp(): + now = datetime.now(pytz.utc).isoformat()[:-13] + return f"{now}Z" -def embed_and_merge_note_items_to_variant_sample(request, embedded_vs): - ''' - Important: Modifies `embedded_vs` in-place. +def get_case_spreadsheet_file_name() -> str: + timestamp = get_timestamp() + return f"case-spreadsheet-filtering-{timestamp}" - This function requires that `embedded_vs` contain the below - `note_containing_fields` with at least a populated `@id` field - (if present). - ''' - note_containing_fields = [ + +def get_items_from_search( + context: CGAPRoot, request: Request, spreadsheet_request: SpreadsheetRequest +) -> Iterator[JsonObject]: + search_to_perform = spreadsheet_request.get_compound_search() + return FilterSetSearch(context, request, search_to_perform).get_search_results() + + +def get_variant_sample_rows( + items_for_spreadsheet: Iterable[JsonObject], + spreadsheet_request: SpreadsheetRequest, + embed_additional_items: Optional[bool] = True, +) -> Iterator[Iterable[str]]: + return VariantSampleSpreadsheet( + items_for_spreadsheet, + spreadsheet_request=spreadsheet_request, + embed_additional_items=embed_additional_items, + ).yield_rows() + + +def get_case_rows( + items_for_spreadsheet: Iterable[JsonObject], +) -> Iterator[Iterable[str]]: + return CaseSpreadsheet(items_for_spreadsheet).yield_rows() + + +def get_spreadsheet_response( + file_name: str, spreadsheet_rows: Iterator[List[str]], file_format: str +) -> Response: + return SpreadsheetGenerator( + file_name, spreadsheet_rows, file_format=file_format + ).get_streaming_response() + + +@dataclass(frozen=True) +class VariantSampleSpreadsheet(SpreadsheetFromColumnTuples): + + HEADER_SPACER_LINE = ["## -------------------------------------------------------"] + NOTE_FIELDS_TO_EMBED = [ "variant.interpretations", "variant.discovery_interpretations", "variant.variant_notes", @@ -132,17 +179,458 @@ def embed_and_merge_note_items_to_variant_sample(request, embedded_vs): "interpretation", "discovery_interpretation", "variant_notes", - "gene_notes" + "gene_notes", ] - # TODO: Parallelize (^ notes per VS)? - for note_field in note_containing_fields: - for incomplete_note_obj in simple_path_ids(embedded_vs, note_field): - # Using request.embed instead of CustomEmbed because we're fine with getting from ES (faster) - # for search-based spreadsheet requests. - note_subreq = make_subrequest(request, incomplete_note_obj["@id"]) - # We don't get _stats on subreq of www-encoded-form POST requests; need to look into (could perhaps amend in snovault) - # Add this in to prevent error in snovault's `after_cursor_execute` - setattr(note_subreq, "_stats", request._stats) - note_response = request.invoke_subrequest(note_subreq) - incomplete_note_obj.update(note_response.json) + embed_additional_items: bool = True + spreadsheet_request: Optional[SpreadsheetRequest] = None + + def _get_headers(self) -> List[List[str]]: + result = [] + result += self._get_available_header_lines() + if result: + result += [self.HEADER_SPACER_LINE] + return result + + def _get_available_header_lines(self) -> List[List[str]]: + result = [] + if self.spreadsheet_request: + result += self._get_case_accession_line() + result += self._get_case_title_line() + result += self._get_readable_filters_line() + return result + + def _get_case_accession_line(self) -> List[List[str]]: + result = [] + case_accession = self.spreadsheet_request.get_case_accession() + if case_accession: + result.append(["#", "Case Accession:", "", case_accession]) + return result + + def _get_case_title_line(self) -> List[List[str]]: + result = [] + case_title = self.spreadsheet_request.get_case_title() + if case_title: + result.append(["#", "Case Title:", "", case_title]) + return result + + def _get_readable_filters_line(self) -> List[List[str]]: + result = [] + search = self.spreadsheet_request.get_compound_search() + if search: + readable_filter_blocks = human_readable_filter_block_queries(search) + result.append(["#", "Filters Selected:", "", readable_filter_blocks]) + return result + + def _get_row_for_item(self, item_to_evaluate: JsonObject) -> List[str]: + self._add_embeds(item_to_evaluate) + variant_sample = VariantSample(item_to_evaluate) + return [ + self._evaluate_item_with_column(column, variant_sample) + for column in self._spreadsheet_columns + ] + + def _add_embeds(self, item_to_evaluate: JsonObject) -> None: + if self.embed_additional_items and self.spreadsheet_request: + self._merge_notes(item_to_evaluate) + + def _merge_notes(self, variant_sample_properties: JsonObject) -> None: + """Get Note information not embedded on the VariantSample, and + merge into existing Note in place. + + Assumes the notes are embedded on the VariantSample for all + note fields; this will always be the case for linkTos on the + VariantSample, but may not be for notes on the associated + Variant/Gene. + """ + for note_field in self.NOTE_FIELDS_TO_EMBED: + existing_notes = simple_path_ids(variant_sample_properties, note_field) + for existing_note in existing_notes: + self._update_note(existing_note) + + def _update_note(self, note_properties: JsonObject) -> None: + all_note_properties = self._get_note_by_subrequest(note_properties) + note_properties.update(all_note_properties) + + def _get_note_by_subrequest(self, note_properties: JsonObject) -> JsonObject: + note_identifier = Note(note_properties).get_atid() + request = self.spreadsheet_request.get_request() + return request.embed(note_identifier, as_user=True) + + def _evaluate_item_with_column( + self, + column: SpreadsheetColumn, + variant_sample: VariantSample, + ) -> str: + if column.is_property_evaluator(): + return column.get_field_for_item(variant_sample.get_properties()) + if column.is_callable_evaluator(): + return column.get_field_for_item(variant_sample) + raise SpreadsheetCreationError("Unable to use column for evaluating item") + + @classmethod + def _get_column_tuples(cls) -> List[OrderedSpreadsheetColumn]: + return [ + ("ID", "URL path to the variant", "@id"), + ("Chrom (hg38)", "Chromosome (hg38)", "variant.CHROM"), + ("Pos (hg38)", "Start position (hg38)", "variant.POS"), + ("Chrom (hg19)", "Chromosome (hg19)", "variant.hg19_chr"), + ("Pos (hg19)", "Start position (hg19)", "variant.hg19_pos"), + ("Ref", "Reference Nucleotide", "variant.REF"), + ("Alt", "Alternate Nucleotide", "variant.ALT"), + ( + "Proband genotype", + "Proband Genotype", + "associated_genotype_labels.proband_genotype_label", + ), + ( + "Mother genotype", + "Mother Genotype", + "associated_genotype_labels.mother_genotype_label", + ), + ( + "Father genotype", + "Father Genotype", + "associated_genotype_labels.father_genotype_label", + ), + ("HGVSG", "HGVS genomic nomenclature", "variant.hgvsg"), + ( + "HGVSC", + "HGVS cPos nomenclature", + "variant.genes.genes_most_severe_hgvsc", + ), + ( + "HGVSP", + "HGVS pPos nomenclature", + "variant.genes.genes_most_severe_hgvsp", + ), + ("dbSNP ID", "dbSNP ID of variant", "variant.ID"), + ( + "Genes", + "Gene symbol(s)", + "variant.genes.genes_most_severe_gene.display_title", + ), + ( + "Canonical transcript ID", + "Ensembl ID of canonical transcript of gene variant is in", + cls._get_canonical_transcript_feature, + ), + ( + "Canonical transcript location", + ( + "Number of exon or intron variant is located in canonical" + " transcript, out of total" + ), + cls._get_canonical_transcript_location, + ), + ( + "Canonical transcript coding effect", + "Coding effect of variant in canonical transcript", + cls._get_canonical_transcript_consequence_names, + ), + ( + "Most severe transcript ID", + "Ensembl ID of transcript with worst annotation for variant", + cls._get_most_severe_transcript_feature, + ), + ( + "Most severe transcript location", + ( + "Number of exon or intron variant is located in most severe" + " transcript, out of total" + ), + cls._get_most_severe_transcript_location, + ), + ( + "Most severe transcript coding effect", + "Coding effect of variant in most severe transcript", + cls._get_most_severe_transcript_consequence_names, + ), + ("Inheritance modes", "Inheritance Modes of variant", "inheritance_modes"), + ("NovoPP", "Novocaller Posterior Probability", "novoPP"), + ( + "Cmphet mate", + ( + "Variant ID of mate, if variant is part of a compound heterozygous" + " group" + ), + "cmphet.comhet_mate_variant", + ), + ("Variant Quality", "Variant call quality score", "QUAL"), + ("Genotype Quality", "Genotype call quality score", "GQ"), + ("Strand Bias", "Strand bias estimated using Fisher's exact test", "FS"), + ("Allele Depth", "Number of reads with variant allele", "AD_ALT"), + ("Read Depth", "Total number of reads at position", "DP"), + ("clinvar ID", "Clinvar ID of variant", "variant.csq_clinvar"), + ( + "gnomADv3 total AF", + "Total allele frequency in gnomad v3 (genomes)", + "variant.csq_gnomadg_af", + ), + ( + "gnomADv3 popmax AF", + "Max. allele frequency in gnomad v3 (genomes)", + "variant.csq_gnomadg_af_popmax", + ), + ( + "gnomADv3 popmax population", + "Population with max. allele frequency in gnomad v3 (genomes)", + cls._get_gnomad_v3_popmax_population, + ), + ( + "gnomADv2 exome total AF", + "Total allele frequency in gnomad v2 (exomes)", + "variant.csq_gnomade2_af", + ), + ( + "gnomADv2 exome popmax AF", + "Max. allele frequency in gnomad v2 (exomes)", + "variant.csq_gnomade2_af_popmax", + ), + ( + "gnomADv2 exome popmax population", + "Population with max. allele frequency in gnomad v2 (exomes)", + cls._get_gnomad_v2_popmax_population, + ), + ("GERP++", "GERP++ score", "variant.csq_gerp_rs"), + ("CADD", "CADD score", "variant.csq_cadd_phred"), + ( + "phyloP-30M", + "phyloP (30 Mammals) score", + "variant.csq_phylop30way_mammalian", + ), + ( + "phyloP-100V", + "phyloP (100 Vertebrates) score", + "variant.csq_phylop100way_vertebrate", + ), + ( + "phastCons-100V", + "phastCons (100 Vertebrates) score", + "variant.csq_phastcons100way_vertebrate", + ), + ("SIFT", "SIFT prediction", "variant.csq_sift_pred"), + ("PolyPhen2", "PolyPhen2 prediction", "variant.csq_polyphen2_hvar_pred"), + ("PrimateAI", "Primate AI prediction", "variant.csq_primateai_pred"), + ("REVEL", "REVEL score", "variant.csq_revel_score"), + ("SpliceAI", "SpliceAI score", "variant.spliceaiMaxds"), + ( + "LOEUF", + "Loss-of-function observed/expected upper bound fraction", + "variant.genes.genes_most_severe_gene.oe_lof_upper", + ), + ( + "S-het", + ( + "Estimates of heterozygous selection (source: Cassa et al 2017 Nat" + " Genet doi:10.1038/ng.3831)" + ), + "variant.genes.genes_most_severe_gene.s_het", + ), + ( + "ACMG classification (current)", + "ACMG classification for variant in this case", + "interpretation.classification", + ), + ( + "ACMG rules (current)", + "ACMG rules invoked for variant in this case", + "interpretation.acmg_rules_invoked.acmg_rule_name", + ), + ( + "Clinical interpretation notes (current)", + "Clinical interpretation notes written for this case", + "interpretation.note_text", + ), + ( + "Gene candidacy (current)", + "Gene candidacy level selected for this case", + "discovery_interpretation.gene_candidacy", + ), + ( + "Variant candidacy (current)", + "Variant candidacy level selected for this case", + "discovery_interpretation.variant_candidacy", + ), + ( + "Discovery notes (current)", + "Gene/variant discovery notes written for this case", + "discovery_interpretation.note_text", + ), + ( + "Variant notes (current)", + "Additional notes on variant written for this case", + "variant_notes.note_text", + ), + ( + "Gene notes (current)", + "Additional notes on gene written for this case", + "gene_notes.note_text", + ), + ( + "ACMG classification (previous)", + "ACMG classification for variant in previous cases", + cls._get_note_of_same_project( + "variant.interpretations", "classification" + ), + ), + ( + "ACMG rules (previous)", + "ACMG rules invoked for variant in previous cases", + cls._get_note_of_same_project( + "variant.interpretations", "acmg_rules_invoked.acmg_rule_name" + ), + ), + ( + "Clinical interpretation (previous)", + "Clinical interpretation notes written for previous cases", + cls._get_note_of_same_project("variant.interpretations", "note_text"), + ), + ( + "Gene candidacy (previous)", + "Gene candidacy level selected for previous cases", + cls._get_note_of_same_project( + "variant.discovery_interpretations", "gene_candidacy" + ), + ), + ( + "Variant candidacy (previous)", + "Variant candidacy level selected for previous cases", + cls._get_note_of_same_project( + "variant.discovery_interpretations", "variant_candidacy" + ), + ), + ( + "Discovery notes (previous)", + "Gene/variant discovery notes written for previous cases", + cls._get_note_of_same_project( + "variant.discovery_interpretations", "note_text" + ), + ), + ( + "Variant notes (previous)", + "Additional notes on variant written for previous cases", + cls._get_note_of_same_project("variant.variant_notes", "note_text"), + ), + ( + "Gene notes (previous)", + "Additional notes on gene written for previous cases", + cls._get_note_of_same_project( + "variant.genes.genes_most_severe_gene.gene_notes", "note_text" + ), + ), + ] + + @classmethod + def _get_canonical_transcript_feature(cls, variant_sample: VariantSample) -> str: + return variant_sample.get_canonical_transcript_feature() + + @classmethod + def _get_canonical_transcript_location(cls, variant_sample: VariantSample) -> str: + return variant_sample.get_canonical_transcript_location() + + @classmethod + def _get_canonical_transcript_consequence_names( + cls, variant_sample: VariantSample + ) -> str: + return variant_sample.get_canonical_transcript_consequence_names() + + @classmethod + def _get_most_severe_transcript_feature(cls, variant_sample: VariantSample) -> str: + return variant_sample.get_most_severe_transcript_feature() + + @classmethod + def _get_most_severe_transcript_location(cls, variant_sample: VariantSample) -> str: + return variant_sample.get_most_severe_transcript_location() + + @classmethod + def _get_most_severe_transcript_consequence_names( + cls, variant_sample: VariantSample + ) -> str: + return variant_sample.get_most_severe_transcript_consequence_names() + + @classmethod + def _get_gnomad_v3_popmax_population(cls, variant_sample: VariantSample) -> str: + return variant_sample.get_gnomad_v3_popmax_population() + + @classmethod + def _get_gnomad_v2_popmax_population(cls, variant_sample: VariantSample) -> str: + return variant_sample.get_gnomad_v2_popmax_population() + + @classmethod + def _get_note_of_same_project( + cls, note_property_location: str, note_property_to_retrieve: str + ) -> Callable: + note_evaluator = partial( + cls._get_note_properties, + note_property_location=note_property_location, + note_property_to_retrieve=note_property_to_retrieve, + ) + return note_evaluator + + @classmethod + def _get_note_properties( + cls, + variant_sample: VariantSample, + note_property_location: str = "", + note_property_to_retrieve: str = "", + ) -> str: + result = "" + note = variant_sample.get_most_recent_note_of_same_project_from_property( + note_property_location + ) + if note: + result = get_values_for_field( + note.get_properties(), note_property_to_retrieve + ) + return result + + +@dataclass(frozen=True) +class CaseSpreadsheet(SpreadsheetFromColumnTuples): + + NO_FLAG_DEFAULT = "No flag" + + def _get_headers(self) -> List[str]: + return [] + + def _get_row_for_item(self, item_to_evaluate: JsonObject) -> List[str]: + return [ + column.get_field_for_item(item_to_evaluate) + for column in self._spreadsheet_columns + ] + + @classmethod + def _get_column_tuples(cls) -> List[OrderedSpreadsheetColumn]: + return [ + ("Case ID", "Case identifier", "case_title"), + ("UUID", "Unique database identifier", "uuid"), + ("Individual ID", "Individual identifier", "individual.individual_id"), + ("Individual sex", "Sex of associated individual", "individual.sex"), + ("Proband case", "Whether case is for a proband", "proband_case"), + ("Family ID", "Family identifier", "family.family_id"), + ("Analysis type", "Analysis type", "sample_processing.analysis_type"), + ("Sample ID", "Primary sample identifier", "sample.display_title"), + ("Sequencing", "Primary sample sequencing type", "sample.workup_type"), + ("QC flag", "Overall QC flag", cls._get_qc_flag), + ( + "Completed QC", + "Completed QC steps", + "quality_control_flags.completed_qcs", + ), + ( + "QC warnings", + "QC steps with warning flags", + "sample_processing.quality_control_metrics.warn", + ), + ( + "QC failures", + "QC steps with failure flags", + "sample_processing.quality_control_metrics.fail", + ), + ] + + @classmethod + def _get_qc_flag(cls, item_to_evaluate: JsonObject) -> str: + qc_flag = get_values_for_field(item_to_evaluate, "quality_control_flags.flag") + return qc_flag or cls.NO_FLAG_DEFAULT diff --git a/src/encoded/batch_download_utils.py b/src/encoded/batch_download_utils.py index be79d0d754..576865c9ae 100644 --- a/src/encoded/batch_download_utils.py +++ b/src/encoded/batch_download_utils.py @@ -1,48 +1,49 @@ import csv +import json +from abc import ABC, abstractmethod +from dataclasses import dataclass +from functools import lru_cache +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Tuple, + Union, +) from urllib.parse import parse_qs + import structlog -from snovault.util import simple_path_ids # , debug_log +from pyramid.request import Request +from pyramid.response import Response +from snovault.util import simple_path_ids +from .root import CGAPRoot +from .search.compound_search import GLOBAL_FLAGS, INTERSECT, CompoundSearchBuilder +from .types.base import Item +from .types.filter_set import FILTER_BLOCKS +from .util import JsonObject log = structlog.getLogger(__name__) -# Unsure if we might prefer the below approach to avoid recursion or not- -# def simple_path_ids(obj, path): -# if isinstance(path, str): -# path = path.split('.') -# path.reverse() -# value = None -# curr_obj_q = [] -# if isinstance(obj, list): -# curr_obj_q = obj -# else: -# curr_obj_q = [obj] -# while len(path) > 0: -# name = path.pop() -# next_q = [] -# for curr_obj in curr_obj_q: -# value = curr_obj.get(name, None) -# if value is None: -# continue -# if not isinstance(value, list): -# value = [value] -# for v in value: -# next_q.append(v) -# curr_obj_q = next_q -# else: -# return curr_obj_q - - -############################## -### Spreadsheet Generation ### -############################## +OrderedSpreadsheetColumn = Tuple[str, str, Union[str, Callable]] + +CSV_EXTENSION = "csv" +TSV_EXTENSION = "tsv" +ACCEPTABLE_FILE_FORMATS = set([TSV_EXTENSION, CSV_EXTENSION]) +DEFAULT_FILE_FORMAT = TSV_EXTENSION +FILE_FORMAT_TO_DELIMITER = {CSV_EXTENSION: ",", TSV_EXTENSION: "\t"} def get_values_for_field(item, field, remove_duplicates=True): """Copied over from 4DN / batch_download / metadata.tsv endpoint code""" c_value = [] - if remove_duplicates: for value in simple_path_ids(item, field): str_value = str(value) @@ -51,61 +52,36 @@ def get_values_for_field(item, field, remove_duplicates=True): else: for value in simple_path_ids(item, field): c_value.append(str(value)) - return ", ".join(c_value) -def convert_item_to_sheet_dict(item, spreadsheet_mappings): - ''' - We assume we have @@embedded representation of Item here - that has all fields required by spreadsheet_mappings, either - through an /embed request or @@embedded representation having - proper embedded_list. - ''' - - if '@id' not in item: - return None - - sheet_dict = {} # OrderedDict() # Keyed by column title. Maybe OrderedDict not necessary now.. - - for column_title, cgap_field_or_func, description in spreadsheet_mappings: - if cgap_field_or_func is None: # Skip - continue - - is_field_str = isinstance(cgap_field_or_func, str) - - if not is_field_str: # Assume render or custom-logic function - sheet_dict[column_title] = cgap_field_or_func(item) - else: - sheet_dict[column_title] = get_values_for_field(item, cgap_field_or_func) - - return sheet_dict - - def human_readable_filter_block_queries(filterset_blocks_request): parsed_filter_block_qs = [] - fb_len = len(filterset_blocks_request["filter_blocks"]) - for fb in filterset_blocks_request["filter_blocks"]: + fb_len = len(filterset_blocks_request[FILTER_BLOCKS]) + for fb in filterset_blocks_request[FILTER_BLOCKS]: curr_fb_str = None - if not fb["query"]: + query = fb.get("query") + if not query: curr_fb_str = "" if fb_len > 1: curr_fb_str = "( " + curr_fb_str + " )" else: - qs_dict = parse_qs(fb["query"]) + qs_dict = parse_qs(query) curr_fb_q = [] for field, value in qs_dict.items(): formstr = field + " = " if len(value) == 1: formstr += str(value[0]) else: - formstr += "[ " + " | ".join([ str(v) for v in value ]) + " ]" + formstr += "[ " + " | ".join([str(v) for v in value]) + " ]" curr_fb_q.append(formstr) curr_fb_str = " & ".join(curr_fb_q) if fb_len > 1: curr_fb_str = "( " + curr_fb_str + " )" parsed_filter_block_qs.append(curr_fb_str) - return (" AND " if filterset_blocks_request.get("intersect", False) else " OR ").join(parsed_filter_block_qs) + return (" AND " if filterset_blocks_request.get(INTERSECT, False) else " OR ").join( + parsed_filter_block_qs + ) class Echo(object): @@ -113,67 +89,229 @@ def write(self, line): return line.encode("utf-8") +class SpreadsheetCreationError(Exception): + pass -def stream_tsv_output( - dictionaries_iterable, - spreadsheet_mappings, - file_format = "tsv", - header_rows=None -): - ''' - Generator which converts iterable of column:value dictionaries into a TSV stream. - :param dictionaries_iterable: Iterable of dictionaries, each containing TSV_MAPPING keys and values from a file in ExperimentSet. - ''' - writer = csv.writer( - Echo(), - delimiter= "\t" if file_format == "tsv" else ",", - quoting=csv.QUOTE_NONNUMERIC - ) +@dataclass(frozen=True) +class SpreadsheetColumn: - # yield writer.writerow("\xEF\xBB\xBF") # UTF-8 BOM - usually shows up as special chars (not useful) + title: str + description: str + evaluator: Union[str, Callable] - # Header/Intro Rows (if any) - for row in (header_rows or []): - yield writer.writerow(row) + def get_title(self) -> str: + return self.title - ## Add in headers (column title) and descriptions - title_headers = [] - description_headers = [] - for column_title, cgap_field_or_func, description in spreadsheet_mappings: - title_headers.append(column_title) - description_headers.append(description) + def get_description(self) -> str: + return self.description - # Prepend comment hash in case people using this spreadsheet file programmatically. - title_headers[0] = "# " + title_headers[0] - description_headers[0] = "# " + description_headers[0] + def get_evaluator(self) -> Union[str, Callable]: + return self.evaluator - yield writer.writerow(title_headers) - yield writer.writerow(description_headers) + def get_field_for_item(self, item: Any) -> str: + if self.is_property_evaluator() and isinstance(item, Dict): + return self._get_field_from_item(item) + if self.is_callable_evaluator(): + return self.evaluator(item) + raise SpreadsheetCreationError( + f"Unable to evaluate item {item} with evaluator {self.evaluator}" + ) - del title_headers - del description_headers + def _get_field_from_item(self, item: Any) -> str: + return get_values_for_field(item, self.evaluator) - for vs_dict in dictionaries_iterable: - if vs_dict is None: # No view permissions (?) - row = [ "" for sm in spreadsheet_mappings ] - row[0] = "# Not Available" - yield writer.writerow(row) - else: - # print("Printing", vs_dict) - row = [ vs_dict.get(sm[0]) or "" for sm in spreadsheet_mappings ] - yield writer.writerow(row) - - -# TODO: Fortunately, I don't see any uses of this function. -kmp 25-Sep-2022 -# -# def build_xlsx_spreadsheet(dictionaries_iterable, spreadsheet_mappings): -# """TODO""" -# from tempfile import NamedTemporaryFile -# from openpyxl import Workbook -# wb = Workbook() -# -# with NamedTemporaryFile() as tmp: -# wb.save(tmp.name) -# tmp.seek(0) -# stream = tmp.read() # TODO: Seems like you'd want to return this value? -kmp 25-Sep-2022 + def is_property_evaluator(self): + return isinstance(self.evaluator, str) + + def is_callable_evaluator(self): + return callable(self.evaluator) + + +@dataclass(frozen=True) +class SpreadsheetTemplate(ABC): + + items_to_evaluate: Iterable[JsonObject] + + @abstractmethod + def _get_headers(self) -> List[List[str]]: + pass + + @abstractmethod + def _get_column_titles(self) -> List[str]: + pass + + @abstractmethod + def _get_column_descriptions(self) -> List[str]: + pass + + @abstractmethod + def _get_row_for_item(self, item_to_evaluate: JsonObject) -> List[str]: + pass + + def yield_rows(self) -> Iterator[Iterable[str]]: + yield from self._yield_headers() + yield from self._yield_column_rows() + yield from self._yield_item_rows() + + def _yield_headers(self) -> Iterator[Iterable[str]]: + for header in self._get_headers(): + yield header + + def _yield_column_rows(self) -> Iterator[Iterable[str]]: + yield self._get_column_descriptions() + yield self._get_column_titles() + + def _yield_item_rows(self) -> Iterator[Iterable[str]]: + for item in self.items_to_evaluate: + yield self._get_row_for_item(item) + + +@dataclass(frozen=True) +class SpreadsheetFromColumnTuples(SpreadsheetTemplate, ABC): + @classmethod + @abstractmethod + def _get_column_tuples(cls) -> None: + pass + + @property + def _spreadsheet_columns(self) -> None: + return self.get_spreadsheet_columns() + + @classmethod + @lru_cache() + def get_spreadsheet_columns(cls) -> List[SpreadsheetColumn]: + column_tuples = cls._get_column_tuples() + return cls._convert_column_tuples_to_spreadsheet_columns(column_tuples) + + @classmethod + def _convert_column_tuples_to_spreadsheet_columns( + cls, + columns: Iterable[OrderedSpreadsheetColumn], + ) -> List[SpreadsheetColumn]: + return [SpreadsheetColumn(*column) for column in columns] + + def _get_column_titles(self) -> List[str]: + return [column.get_title() for column in self._spreadsheet_columns] + + def _get_column_descriptions(self) -> List[str]: + result = [] + for idx, column in enumerate(self._spreadsheet_columns): + description = column.get_description() + if idx == 0: + result.append(f"# {description}") + else: + result.append(description) + return result + + +@dataclass(frozen=True) +class SpreadsheetRequest: + + CASE_ACCESSION = "case_accession" + CASE_TITLE = "case_title" + COMPOUND_SEARCH_REQUEST = "compound_search_request" + FILE_FORMAT = "file_format" + + request: Request + + def get_request(self) -> Request: + return self.request + + @property + def parameters(self) -> JsonObject: + return self.request.params or self.request.json + + def get_file_format(self) -> str: + return self.parameters.get(self.FILE_FORMAT, DEFAULT_FILE_FORMAT).lower() + + def get_case_accession(self) -> str: + return self.parameters.get(self.CASE_ACCESSION, "") + + def get_case_title(self) -> str: + return self.parameters.get(self.CASE_TITLE, "") + + def get_compound_search(self) -> JsonObject: + compound_search = self.parameters.get(self.COMPOUND_SEARCH_REQUEST, {}) + if isinstance(compound_search, str): + compound_search = json.loads(compound_search) + return compound_search + + +@dataclass(frozen=True) +class FilterSetSearch: + + context: Union[CGAPRoot, Item] + request: Request + compound_search: Mapping + + def get_search_results(self) -> Iterator[JsonObject]: + return CompoundSearchBuilder.execute_filter_set( + self.context, + self.request, + self._get_filter_set(), + to=CompoundSearchBuilder.ALL, + global_flags=self._get_global_flags(), + intersect=self._is_intersect(), + return_generator=True, + ) + + def _get_filter_set(self) -> JsonObject: + return CompoundSearchBuilder.extract_filter_set_from_search_body( + self.request, self.compound_search + ) + + def _get_global_flags(self) -> Union[str, None]: + return self.compound_search.get(GLOBAL_FLAGS) + + def _is_intersect(self) -> bool: + return bool(self._get_intersect()) + + def _get_intersect(self) -> str: + return self.compound_search.get(INTERSECT, "") + + +@dataclass(frozen=True) +class SpreadsheetGenerator: + + file_name: str + rows_to_write: Iterable[Sequence[str]] + file_format: Optional[str] = DEFAULT_FILE_FORMAT + + def get_streaming_response(self) -> Response: + return Response( + app_iter=self._stream_spreadsheet(), headers=self._get_response_headers() + ) + + def _stream_spreadsheet(self) -> Iterator[Callable]: + writer = self._get_writer() + for row in self.rows_to_write: + if row: + yield writer.writerow(row) + + def _get_writer(self) -> csv.writer: + """Use csv.writer for formatting lines, not writing to file.""" + delimiter = self._get_delimiter() + return csv.writer(Echo(), delimiter=delimiter, quoting=csv.QUOTE_NONNUMERIC) + + def _get_delimiter(self) -> str: + result = FILE_FORMAT_TO_DELIMITER.get(self.file_format) + if result is None: + raise SpreadsheetCreationError( + f"No known delimiter for given file format {self.file_format}" + ) + return result + + def _get_response_headers(self) -> Dict: + return { + "X-Accel-Buffering": "no", + "Content-Disposition": ( + f"attachment; filename={self._get_file_name_with_extension()}" + ), + "Content-Type": f"text/{self.file_format}", + "Content-Description": "File Transfer", + "Cache-Control": "no-store", + } + + def _get_file_name_with_extension(self) -> str: + return f"{self.file_name}.{self.file_format}" diff --git a/src/encoded/custom_embed.py b/src/encoded/custom_embed.py index 7687405996..0eeb46a7a4 100644 --- a/src/encoded/custom_embed.py +++ b/src/encoded/custom_embed.py @@ -26,6 +26,7 @@ ] FORBIDDEN_MSG = {"error": "no view permissions"} DATABASE_ITEM_KEY = "@type" # Key specific to JSON objects that are CGAP items +REQUESTED_FIELDS = "requested_fields" def includeme(config): @@ -43,7 +44,7 @@ def __init__(self, request, item, embed_props): self.ignored_embeds = embed_props.get("ignored_embeds", []) self.desired_embeds = embed_props.get("desired_embeds", []) self.embed_depth = embed_props.get("embed_depth", 4) - self.requested_fields = embed_props.get("requested_fields", []) + self.requested_fields = embed_props.get(REQUESTED_FIELDS, []) self.cache = {} self.invalid_ids = [] @@ -55,6 +56,9 @@ def __init__(self, request, item, embed_props): depth = -1 self.result = self.embed(item, depth) + def get_embedded_item(self): + return self.result + def add_actions(self, item): """ Add the "actions" field to an item according to the request's @@ -380,7 +384,7 @@ def embed(context, request): "ignored_embeds": ignored_embeds, "desired_embeds": desired_embeds, "embed_depth": embed_depth, - "requested_fields": requested_fields, + REQUESTED_FIELDS: requested_fields, } for item_id in ids: item_embed = CustomEmbed(request, item_id, embed_props) diff --git a/src/encoded/item_models.py b/src/encoded/item_models.py new file mode 100644 index 0000000000..1a177e6ba3 --- /dev/null +++ b/src/encoded/item_models.py @@ -0,0 +1,503 @@ +import math +from dataclasses import dataclass +from typing import Iterable, List, Union + +from snovault.util import simple_path_ids + +from .util import JsonObject + + +LinkTo = Union[str, JsonObject] + + +@dataclass(frozen=True) +class Item: + ATID = "@id" + PROJECT = "project" + + properties: JsonObject + + @property + def _atid(self) -> str: + return self.properties.get(self.ATID, "") + + @property + def _project(self) -> LinkTo: + return self.properties.get(self.PROJECT, "") + + def get_properties(self) -> JsonObject: + return self.properties + + def get_atid(self) -> str: + return self._atid + + def get_project(self) -> LinkTo: + return self._project + + +@dataclass(frozen=True) +class VariantConsequence(Item): + # Schema constants + IMPACT = "impact" + IMPACT_HIGH = "HIGH" + IMPACT_LOW = "LOW" + IMPACT_MODERATE = "MODERATE" + IMPACT_MODIFIER = "MODIFIER" + VAR_CONSEQ_NAME = "var_conseq_name" + + DOWNSTREAM_GENE_CONSEQUENCE = "downstream_gene_variant" + FIVE_PRIME_UTR_CONSEQUENCE = "5_prime_UTR_variant" + THREE_PRIME_UTR_CONSEQUENCE = "3_prime_UTR_variant" + UPSTREAM_GENE_CONSEQUENCE = "upstream_gene_variant" + + @property + def _name(self) -> str: + return self.properties.get(self.VAR_CONSEQ_NAME, "") + + @property + def _impact(self) -> str: + return self.properties.get(self.IMPACT, "") + + def get_name(self) -> str: + return self._name + + def get_impact(self) -> str: + return self._impact + + def is_downstream(self) -> str: + return self._name == self.DOWNSTREAM_GENE_CONSEQUENCE + + def is_upstream(self) -> str: + return self._name == self.UPSTREAM_GENE_CONSEQUENCE + + def is_three_prime_utr(self) -> str: + return self._name == self.THREE_PRIME_UTR_CONSEQUENCE + + def is_five_prime_utr(self) -> str: + return self._name == self.FIVE_PRIME_UTR_CONSEQUENCE + + +@dataclass(frozen=True) +class Transcript: + # Schema constants + CSQ_CANONICAL = "csq_canonical" + CSQ_CONSEQUENCE = "csq_consequence" + CSQ_DISTANCE = "csq_distance" + CSQ_EXON = "csq_exon" + CSQ_FEATURE = "csq_feature" + CSQ_INTRON = "csq_intron" + CSQ_MOST_SEVERE = "csq_most_severe" + + # Class constants + LOCATION_EXON = "Exon" + LOCATION_INTRON = "Intron" + LOCATION_DOWNSTREAM = "bp downstream" + LOCATION_UPSTREAM = "bp upstream" + LOCATION_FIVE_PRIME_UTR = "5' UTR" + LOCATION_THREE_PRIME_UTR = "3' UTR" + IMPACT_RANKING = { + VariantConsequence.IMPACT_HIGH: 0, + VariantConsequence.IMPACT_MODERATE: 1, + VariantConsequence.IMPACT_LOW: 2, + VariantConsequence.IMPACT_MODIFIER: 3, + } + + properties: JsonObject + + @property + def _canonical(self) -> bool: + return self.properties.get(self.CSQ_CANONICAL, False) + + @property + def _most_severe(self) -> bool: + return self.properties.get(self.CSQ_MOST_SEVERE, False) + + @property + def _exon(self) -> str: + return self.properties.get(self.CSQ_EXON, "") + + @property + def _intron(self) -> str: + return self.properties.get(self.CSQ_INTRON, "") + + @property + def _distance(self) -> str: + return self.properties.get(self.CSQ_DISTANCE, "") + + @property + def _feature(self) -> str: + return self.properties.get(self.CSQ_FEATURE, "") + + @property + def _consequences(self) -> List[LinkTo]: + return self.properties.get(self.CSQ_CONSEQUENCE, []) + + def is_canonical(self) -> bool: + return self._canonical + + def is_most_severe(self) -> bool: + return self._most_severe + + def get_feature(self) -> str: + return self._feature + + def get_location(self) -> str: + result = "" + most_severe_consequence = self._get_most_severe_consequence() + if most_severe_consequence: + result = self._get_location_by_most_severe_consequence( + most_severe_consequence + ) + return result + + def _get_most_severe_consequence(self) -> Union[VariantConsequence, None]: + result = None + most_severe_rank = math.inf + for consequence in self._get_consequences(): + impact = consequence.get_impact() + impact_rank = self.IMPACT_RANKING.get(impact, math.inf) + if impact_rank < most_severe_rank: + most_severe_rank = impact_rank + result = consequence + return result + + def _get_consequences(self) -> List[VariantConsequence]: + return [ + VariantConsequence(item) + for item in self._consequences + if isinstance(item, dict) + ] + + def _get_location_by_most_severe_consequence( + self, most_severe_consequence: VariantConsequence + ) -> str: + if self._exon: + return self._get_exon_location(most_severe_consequence) + if self._intron: + return self._get_intron_location(most_severe_consequence) + if self._distance: + return self._get_distance_location(most_severe_consequence) + return "" + + def _get_exon_location(self, consequence: VariantConsequence) -> str: + location = f"{self.LOCATION_EXON} {self._exon}" + return self._add_utr_suffix_if_needed(location, consequence) + + def _get_intron_location(self, consequence: VariantConsequence) -> str: + location = f"{self.LOCATION_INTRON} {self._intron}" + return self._add_utr_suffix_if_needed(location, consequence) + + def _get_distance_location(self, consequence: VariantConsequence) -> str: + if consequence.is_upstream(): + return f"{self._distance} {self.LOCATION_UPSTREAM}" + if consequence.is_downstream(): + return f"{self._distance} {self.LOCATION_DOWNSTREAM}" + return "" + + def _add_utr_suffix_if_needed( + self, location: str, consequence: VariantConsequence + ) -> str: + if consequence.is_three_prime_utr(): + return self._add_three_prime_utr_suffix(location) + if consequence.is_five_prime_utr(): + return self._add_five_prime_utr_suffix(location) + return location + + def _add_three_prime_utr_suffix(self, location: str) -> str: + return self._add_utr_suffix(location, self.LOCATION_THREE_PRIME_UTR) + + def _add_five_prime_utr_suffix(self, location: str) -> str: + return self._add_utr_suffix(location, self.LOCATION_FIVE_PRIME_UTR) + + def _add_utr_suffix(self, location: str, utr_suffix: str) -> str: + if location: + return f"{location} ({utr_suffix})" + return utr_suffix + + def get_consequence_names(self) -> str: + return ", ".join( + [consequence.get_name() for consequence in self._get_consequences()] + ) + + +@dataclass(frozen=True) +class Variant(Item): + + # Schema constants + CSQ_CANONICAL = "csq_canonical" + CSQ_CONSEQUENCE = "csq_consquence" + CSQ_FEATURE = "csq_feature" + CSQ_GNOMADE2_AF_POPMAX = "csq_gnomade2_af_popmax" + CSQ_GNOMADG_AF_POPMAX = "csq_gnomadg_af_popmax" + CSQ_MOST_SEVERE = "csq_most_severe" + DISTANCE = "distance" + EXON = "exon" + INTRON = "intron" + MOST_SEVERE_LOCATION = "most_severe_location" + TRANSCRIPT = "transcript" + + GNOMAD_V2_AF_PREFIX = "csq_gnomade2_af-" + GNOMAD_V3_AF_PREFIX = "csq_gnomadg_af-" + GNOMAD_POPULATION_SUFFIX_TO_NAME = { + "afr": "African-American/African", + "ami": "Amish", + "amr": "Latino", + "asj": "Ashkenazi Jewish", + "eas": "East Asian", + "fin": "Finnish", + "mid": "Middle Eastern", + "nfe": "Non-Finnish European", + "oth": "Other Ancestry", + "sas": "South Asian", + } + + @property + def _transcripts(self) -> List[JsonObject]: + return self.properties.get(self.TRANSCRIPT, []) + + @property + def _most_severe_location(self) -> str: + return self.properties.get(self.MOST_SEVERE_LOCATION, "") + + @property + def _csq_gnomadg_af_popmax(self) -> Union[float, None]: + return self.properties.get(self.CSQ_GNOMADG_AF_POPMAX) + + @property + def _csq_gnomade2_af_popmax(self) -> Union[float, None]: + return self.properties.get(self.CSQ_GNOMADE2_AF_POPMAX) + + def get_most_severe_location(self) -> str: + return self._most_severe_location + + def _get_transcripts(self) -> List[Transcript]: + return [Transcript(transcript) for transcript in self._transcripts] + + def _get_canonical_transcript(self) -> Union[Transcript, None]: + for transcript in self._get_transcripts(): + if transcript.is_canonical(): + return transcript + + def _get_most_severe_transcript(self) -> Union[Transcript, None]: + for transcript in self._get_transcripts(): + if transcript.is_most_severe(): + return transcript + + def get_canonical_transcript_feature(self) -> str: + canonical_transcript = self._get_canonical_transcript() + if canonical_transcript: + return canonical_transcript.get_feature() + return "" + + def get_most_severe_transcript_feature(self) -> str: + most_severe_transcript = self._get_most_severe_transcript() + if most_severe_transcript: + return most_severe_transcript.get_feature() + return "" + + def get_canonical_transcript_consequence_names(self) -> str: + canonical_transcript = self._get_canonical_transcript() + if canonical_transcript: + return canonical_transcript.get_consequence_names() + return "" + + def get_most_severe_transcript_consequence_names(self) -> str: + most_severe_transcript = self._get_most_severe_transcript() + if most_severe_transcript: + return most_severe_transcript.get_consequence_names() + return "" + + def get_canonical_transcript_location(self) -> str: + canonical_transcript = self._get_canonical_transcript() + if canonical_transcript: + return canonical_transcript.get_location() + return "" + + def get_most_severe_transcript_location(self) -> str: + most_severe_transcript = self._get_most_severe_transcript() + if most_severe_transcript: + return most_severe_transcript.get_location() + return "" + + def get_gnomad_v3_popmax_population(self) -> str: + result = "" + gnomad_v3_af_popmax = self._csq_gnomadg_af_popmax + if gnomad_v3_af_popmax: + result = self._get_gnomad_v3_population_for_allele_fraction( + gnomad_v3_af_popmax + ) + return result + + def get_gnomad_v2_popmax_population(self) -> str: + result = "" + gnomad_v2_af_popmax = self._csq_gnomade2_af_popmax + if gnomad_v2_af_popmax: + result = self._get_gnomad_v2_population_for_allele_fraction( + gnomad_v2_af_popmax + ) + return result + + def _get_gnomad_v3_population_for_allele_fraction( + self, allele_fraction: float + ) -> str: + return self._get_gnomad_population_for_allele_fraction( + self.GNOMAD_V3_AF_PREFIX, allele_fraction + ) + + def _get_gnomad_v2_population_for_allele_fraction( + self, allele_fraction: float + ) -> str: + return self._get_gnomad_population_for_allele_fraction( + self.GNOMAD_V2_AF_PREFIX, allele_fraction + ) + + def _get_gnomad_population_for_allele_fraction( + self, gnomad_af_prefix: str, allele_fraction: float + ) -> str: + result = "" + for ( + gnomad_suffix, + population_name, + ) in self.GNOMAD_POPULATION_SUFFIX_TO_NAME.items(): + population_property_name = gnomad_af_prefix + gnomad_suffix + allele_frequency = self.properties.get(population_property_name) + if allele_frequency == allele_fraction: + result = population_name + break + return result + + +@dataclass(frozen=True) +class Note(Item): + pass + + +@dataclass(frozen=True) +class VariantSample(Item): + + # Schema constants + VARIANT = "variant" + + @property + def _variant(self) -> LinkTo: + return self.properties.get(self.VARIANT, "") + + def _get_variant(self) -> Union[Variant, None]: + if isinstance(self._variant, dict): + return Variant(self._variant) + return + + def get_canonical_transcript_feature(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_canonical_transcript_feature() + return "" + + def get_canonical_transcript_location(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_canonical_transcript_location() + return "" + + def get_canonical_transcript_consequence_names(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_canonical_transcript_consequence_names() + return "" + + def get_most_severe_transcript_feature(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_most_severe_transcript_feature() + return "" + + def get_most_severe_transcript_location(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_most_severe_transcript_location() + return "" + + def get_most_severe_transcript_consequence_names(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_most_severe_transcript_consequence_names() + return "" + + def get_gnomad_v3_popmax_population(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_gnomad_v3_popmax_population() + return "" + + def get_gnomad_v2_popmax_population(self) -> str: + variant = self._get_variant() + if variant: + return variant.get_gnomad_v2_popmax_population() + return "" + + def get_most_recent_note_of_same_project_from_property( + self, note_property_location: str + ) -> Union[Note, None]: + result = None + notes_at_location = list( + simple_path_ids(self.properties, note_property_location) + ) + if notes_at_location: + result = self._get_most_recent_note_of_same_project(notes_at_location) + return result + + def _get_most_recent_note_of_same_project( + self, notes_properties: Iterable[JsonObject] + ) -> Union[Note, None]: + result = None + for note_properties in reversed(notes_properties): + note = Note(note_properties) + if note.get_project() == self.get_project(): + result = note + break + return result + + +@dataclass(frozen=True) +class VariantSampleListSelection: + + VARIANT_SAMPLE_ITEM = "variant_sample_item" + + properties: JsonObject + + @property + def _variant_sample(self) -> LinkTo: + return self.properties.get(self.VARIANT_SAMPLE_ITEM, "") + + def get_variant_sample(self) -> LinkTo: + return self._variant_sample + + +@dataclass(frozen=True) +class VariantSampleList(Item): + + CREATED_FOR_CASE = "created_for_case" + VARIANT_SAMPLES = "variant_samples" + + @property + def _created_for_case(self) -> str: + return self.properties.get(self.CREATED_FOR_CASE, "") + + @property + def _variant_sample_selections(self) -> List[LinkTo]: + return self.properties.get(self.VARIANT_SAMPLES, []) + + def get_associated_case_accession(self) -> str: + return self._created_for_case + + def _get_variant_sample_selections(self) -> List[VariantSampleListSelection]: + return [ + VariantSampleListSelection(variant_sample_selection) + for variant_sample_selection in self._variant_sample_selections + ] + + def get_variant_samples(self) -> List[LinkTo]: + return [ + variant_sample_selection.get_variant_sample() + for variant_sample_selection in self._get_variant_sample_selections() + ] diff --git a/src/encoded/search/compound_search.py b/src/encoded/search/compound_search.py index ae52ace3a3..d289225189 100644 --- a/src/encoded/search/compound_search.py +++ b/src/encoded/search/compound_search.py @@ -1,20 +1,16 @@ import json -# import os import urllib.parse from pyramid.httpexceptions import HTTPBadRequest -# from pyramid.request import Request from pyramid.view import view_config from snovault import TYPES -# from snovault.embed import make_subrequest from snovault.util import debug_log -from ..types.base import get_item_or_none -from ..types.filter_set import FLAGS, FILTER_BLOCKS - from .lucene_builder import LuceneBuilder from .search import SearchBuilder, search as single_query_search from .search_utils import execute_search, build_sort_dicts, make_search_subreq +from ..types.base import get_item_or_none +from ..types.filter_set import FLAGS, FILTER_BLOCKS def includeme(config): @@ -29,6 +25,7 @@ class CompoundSearchBuilder: Entry point is "execute_filter_set". """ + ALL = 'all' TYPE = 'search_type' ID = '@id' QUERY = 'query' @@ -190,7 +187,7 @@ def execute_filter_set(context, request, filter_set, from_=0, to=10, # if we have no filter blocks, there is no context to enable flags, so # pass type_flag + global_flags - if not filter_blocks and flags: + if not filter_blocks and (flags or global_flags): if global_flags: query = cls.combine_query_strings(global_flags, type_flag) else: @@ -353,6 +350,12 @@ def build_query(context, request): return builder.query +GLOBAL_FLAGS = "global_flags" +INTERSECT = "intersect" +FROM = "from" +LIMIT = "limit" + + @view_config(route_name='compound_search', request_method='POST', permission='search') @debug_log def compound_search(context, request): @@ -407,16 +410,16 @@ def compound_search(context, request): body = json.loads(request.body) filter_set = CompoundSearchBuilder.extract_filter_set_from_search_body(request, body) - global_flags = body.get('global_flags', None) - intersect = True if body.get('intersect', False) else False + global_flags = body.get(GLOBAL_FLAGS, None) + intersect = True if body.get(INTERSECT, False) else False # Disabled for time being to allow test(s) to pass. Not sure whether to add Project to FilterSet schema 'search_type' enum. # if filter_set.get(CompoundSearchBuilder.TYPE) not in request.registry[TYPES]["FilterSet"].schema["properties"][CompoundSearchBuilder.TYPE]["enum"]: # raise HTTPBadRequest("Passed bad {} body param: {}".format(CompoundSearchBuilder.TYPE, filter_set.get(CompoundSearchBuilder.TYPE))) - from_ = body.get('from', 0) - limit = body.get('limit', 10) # pagination size 10 works better with ECS - if limit == "all": + from_ = body.get(FROM, 0) + limit = body.get(LIMIT, 10) # pagination size 10 works better with ECS + if limit == CompoundSearchBuilder.ALL: raise HTTPBadRequest("compound_search does not support limit=all at this time.") if limit > 1000: limit = 1000 diff --git a/src/encoded/static/components/item-pages/CaseView/ExportQCMSpreadsheetButton.js b/src/encoded/static/components/item-pages/CaseView/ExportQCMSpreadsheetButton.js new file mode 100644 index 0000000000..0b16819d0f --- /dev/null +++ b/src/encoded/static/components/item-pages/CaseView/ExportQCMSpreadsheetButton.js @@ -0,0 +1,52 @@ +'use strict'; + +import React, { useCallback, useRef } from 'react'; +import PropTypes from 'prop-types'; +import _ from 'underscore'; +import DropdownButton from "react-bootstrap/esm/DropdownButton"; +import DropdownItem from "react-bootstrap/esm/DropdownItem"; + +/** + * Renders a form element with dropdown button. + * Upon click, this component updates form element attributes + * and hidden input elements in response to dropdown option clicked, + * and then submits form. + */ +export const ExportQCMSpreadsheetButton = React.memo(function ExportQCMSpreadsheetButton(props){ + const { requestedCompoundFilterSet, disabled: propDisabled = false } = props; + const formRef = useRef(null); + const onSelect = useCallback(function(eventKey, e){ + // input[name="file_format"] + formRef.current.children[0].value = eventKey; + // `requestedCompoundFilterSet` passed in from VirtualHrefController of the EmbeddedSearchView + // input[name="compound_search_request"] + formRef.current.children[1].value = JSON.stringify(requestedCompoundFilterSet); + formRef.current.submit(); + return false; + }, [ formRef, requestedCompoundFilterSet ]); + + const disabled = propDisabled || !requestedCompoundFilterSet; // TODO: Check if >0 results, as well. + + return ( + // target=_blank causes new tab to open (and then auto-close), but bypasses the isSubmitting onBeforeUnload check in app.js +
+ + + + + TSV spreadsheet + + + CSV spreadsheet + + + XLSX spreadsheet + + +
+ ); +}); +ExportQCMSpreadsheetButton.propTypes = { + disabled: PropTypes.bool, + requestedCompoundFilterSet: PropTypes.object +}; \ No newline at end of file diff --git a/src/encoded/static/components/static-pages/HomePage/UserDashboard.js b/src/encoded/static/components/static-pages/HomePage/UserDashboard.js index e41392cde6..69e1a5a60b 100644 --- a/src/encoded/static/components/static-pages/HomePage/UserDashboard.js +++ b/src/encoded/static/components/static-pages/HomePage/UserDashboard.js @@ -17,6 +17,7 @@ import { AboveTableControlsBaseCGAP } from './../../browse/AboveTableControlsBas import { EmbeddedCaseSearchTable } from './../../item-pages/components/EmbeddedItemSearchTable'; import FeedbackButton from '../../item-pages/components/FeedbackButton'; +import { ExportQCMSpreadsheetButton } from '../../item-pages/CaseView/ExportQCMSpreadsheetButton'; export const UserDashboard = React.memo(function UserDashboard({ windowHeight, windowWidth }){ @@ -90,7 +91,8 @@ const AboveCasesTableOptions = React.memo(function AboveCasesTableOptions(props) context, onFilter, isContextLoading, navigate, sortBy, sortColumns, - hiddenColumns, addHiddenColumn, removeHiddenColumn, columnDefinitions + hiddenColumns, addHiddenColumn, removeHiddenColumn, columnDefinitions, + requestedCompoundFilterSet } = props; const { filters: ctxFilters = null } = context || {}; @@ -161,6 +163,9 @@ const AboveCasesTableOptions = React.memo(function AboveCasesTableOptions(props) Show Only Proband Cases +
+ +
diff --git a/src/encoded/tests/data/workbook-inserts/gene.json b/src/encoded/tests/data/workbook-inserts/gene.json index 7aa43fc961..31d886646f 100644 --- a/src/encoded/tests/data/workbook-inserts/gene.json +++ b/src/encoded/tests/data/workbook-inserts/gene.json @@ -91,7 +91,7 @@ "prev_name": [ "gamma-aminobutyric acid (GABA) A receptor, delta" ], - "rvis_exac": -0.52053282, + "rvis_exac": 0.000601, "spos_hg19": 1950780, "alias_name": [ "GABA(A) receptor, delta" @@ -174,6 +174,10 @@ "trait_association_gwas_pmid": [ "26426971~26426971" ], + "gene_notes": [ + "62759f07-326a-4e94-8dd0-366741292d07", + "0791a219-856d-41b8-bc7a-7dbbc973f848" + ], "uuid": "f6490485-7910-4019-a45e-02d6fb96f3a5" }, { diff --git a/src/encoded/tests/data/workbook-inserts/individual.json b/src/encoded/tests/data/workbook-inserts/individual.json index fa4582d1f5..bf77e26cc3 100644 --- a/src/encoded/tests/data/workbook-inserts/individual.json +++ b/src/encoded/tests/data/workbook-inserts/individual.json @@ -187,7 +187,11 @@ "status": "shared", "accession": "GAPIDISC7R74", "uuid": "5ec91041-78a0-4758-abef-21c7f5fd9f26", + "samples": [ + "hms-dbmi:sample_1" + ], "sex": "M", + "individual_id": "BRCA proband", "is_deceased": false, "disorders": [ { diff --git a/src/encoded/tests/data/workbook-inserts/note_discovery.json b/src/encoded/tests/data/workbook-inserts/note_discovery.json new file mode 100644 index 0000000000..cbb49ea1ef --- /dev/null +++ b/src/encoded/tests/data/workbook-inserts/note_discovery.json @@ -0,0 +1,18 @@ +[ + { + "uuid": "1e1ea605-b3e2-4abb-8677-d647042e4fc4", + "project": "hms-dbmi", + "institution": "hms-dbmi", + "gene_candidacy": "Strong candidate", + "variant_candidacy": "Moderate candidate", + "note_text": "This gene is a real discovery!" + }, + { + "uuid": "e8046b1b-22a4-40fa-8558-4702dae6d822", + "project": "cgap-core", + "institution": "hms-dbmi", + "gene_candidacy": "Moderate candidate", + "variant_candidacy": "Weak candidate", + "note_text": "This gene is not a real discovery..." + } +] diff --git a/src/encoded/tests/data/workbook-inserts/note_interpretation.json b/src/encoded/tests/data/workbook-inserts/note_interpretation.json index e6ed877fab..235dc34bcc 100644 --- a/src/encoded/tests/data/workbook-inserts/note_interpretation.json +++ b/src/encoded/tests/data/workbook-inserts/note_interpretation.json @@ -1,22 +1,41 @@ [ - { - "project": "hms-dbmi", - "institution": "hms-dbmi", - "uuid": "ab5e1c89-4c88-4a3e-a306-d37a12defd8b", - "note_text": "This variant has been reported in the ClinVar database as Unknown Significance.", - "conclusion": "For this reason, the variant has been classified as VUS.", - "status": "in review", - "classification": "Uncertain significance", - "version": 2, - "previous_note": "de5e1c12-4c88-4a3e-a306-d37a12defa6b" - }, - { - "project": "hms-dbmi", - "institution": "hms-dbmi", - "uuid": "de5e1c12-4c88-4a3e-a306-d37a12defa6b", - "note_text": "This variant has not been reported in the ClinVar database.", - "conclusion": "For this reason, the variant has been classified as VUS.", - "status": "in review", - "classification": "Uncertain significance" - } + { + "project": "hms-dbmi", + "institution": "hms-dbmi", + "uuid": "ab5e1c89-4c88-4a3e-a306-d37a12defd8b", + "note_text": "This variant has been reported in the ClinVar database as Unknown Significance.", + "conclusion": "For this reason, the variant has been classified as VUS.", + "status": "in review", + "classification": "Uncertain significance", + "acmg_rules_invoked": [ + { + "acmg_rule_name": "PM1" + } + ], + "version": 2, + "previous_note": "de5e1c12-4c88-4a3e-a306-d37a12defa6b" + }, + { + "project": "hms-dbmi", + "institution": "hms-dbmi", + "uuid": "de5e1c12-4c88-4a3e-a306-d37a12defa6b", + "note_text": "This variant has not been reported in the ClinVar database.", + "conclusion": "For this reason, the variant has been classified as VUS.", + "status": "in review", + "classification": "Uncertain significance" + }, + { + "project": "cgap-core", + "institution": "hms-dbmi", + "uuid": "2f0557f6-7284-4a5f-976d-0f11f24567fd", + "note_text": "This variant has been reported in the ClinVar database as Pathogenic.", + "conclusion": "For this reason, the variant has been classified as Pathogenic.", + "status": "in review", + "classification": "Pathogenic", + "acmg_rules_invoked": [ + { + "acmg_rule_name": "PS1" + } + ] + } ] diff --git a/src/encoded/tests/data/workbook-inserts/note_standard.json b/src/encoded/tests/data/workbook-inserts/note_standard.json new file mode 100644 index 0000000000..3204080830 --- /dev/null +++ b/src/encoded/tests/data/workbook-inserts/note_standard.json @@ -0,0 +1,14 @@ +[ + { + "uuid": "62759f07-326a-4e94-8dd0-366741292d07", + "project": "hms-dbmi", + "institution": "hms-dbmi", + "note_text": "What a note" + }, + { + "uuid": "0791a219-856d-41b8-bc7a-7dbbc973f848", + "project": "cgap-core", + "institution": "hms-dbmi", + "note_text": "What a poor note" + } +] diff --git a/src/encoded/tests/data/workbook-inserts/sample.json b/src/encoded/tests/data/workbook-inserts/sample.json index 7016569aa6..23b085e4ae 100644 --- a/src/encoded/tests/data/workbook-inserts/sample.json +++ b/src/encoded/tests/data/workbook-inserts/sample.json @@ -8,6 +8,7 @@ ], "bam_sample_id": "Sample_ID", "workup_type": "WES", + "specimen_accession": "BRCA_proband_sample", "specimen_type": "peripheral_blood", "processed_files": [ "hms-dbmi:sample_1_bam" diff --git a/src/encoded/tests/data/workbook-inserts/sample_processing.json b/src/encoded/tests/data/workbook-inserts/sample_processing.json index 6f0d6ea973..a4913fefd6 100644 --- a/src/encoded/tests/data/workbook-inserts/sample_processing.json +++ b/src/encoded/tests/data/workbook-inserts/sample_processing.json @@ -54,6 +54,7 @@ "aliases": [ "hms-dbmi:two_samples" ], + "analysis_type": "WES-Group", "project": "12a92962-8265-4fc0-b2f8-cf14f05db58b", "institution": "hms-dbmi", "samples": [ diff --git a/src/encoded/tests/data/workbook-inserts/variant.json b/src/encoded/tests/data/workbook-inserts/variant.json index deed841cd4..a3f27b47b4 100644 --- a/src/encoded/tests/data/workbook-inserts/variant.json +++ b/src/encoded/tests/data/workbook-inserts/variant.json @@ -69,8 +69,6 @@ } ] }, - - { "ID": "rs142404438", "ALT": "A", @@ -81,6 +79,7 @@ { "genes_most_severe_gene": "f6490485-7910-4019-a45e-02d6fb96f3a5", "genes_most_severe_hgvsc": "ENST00000378585.7:c.*384G>A", + "genes_most_severe_hgvsp": "ENST00000378585.7:p.V>A", "genes_most_severe_transcript": "ENST00000378585", "genes_most_severe_consequence": "d2ca7570-10b9-4314-bc2c-fa6d276c21fb", "genes_most_severe_feature_ncbi": "NM_000815.5" @@ -139,8 +138,14 @@ "variantClass": "SNV", "annotation_id": "chr1:2030666G_A", "spliceaiMaxds": 0, + "csq_clinvar": "89991", "csq_cadd_phred": 0.613, "csq_gnomadg_ac": 30, + "csq_gerp_rs": 5.02, + "csq_sift_pred": "T", + "csq_polyphen2_hvar_pred": "D", + "csq_primateai_pred": "D", + "csq_revel_score": 0.013, "csq_gnomadg_af": 0.000197078, "csq_gnomadg_an": 152224, "csq_gnomadg_ac-xx": 16, @@ -193,6 +198,18 @@ "csq_gnomadg_nhomalt-nfe": 0, "csq_gnomadg_nhomalt-oth": 0, "csq_gnomadg_nhomalt-sas": 0, + "csq_gnomade2_af": [ + 0.0002 + ], + "csq_gnomade2_af_popmax": [ + 0.3 + ], + "csq_gnomade2_af-sas": [ + 0.1 + ], + "csq_gnomade2_af-eas": [ + 0.3 + ], "csq_spliceai_pred_dp_ag": 49, "csq_spliceai_pred_dp_al": -37, "csq_spliceai_pred_dp_dg": -13, @@ -205,6 +222,18 @@ "csq_phylop30way_mammalian": -3.49499988555908, "csq_phylop100way_vertebrate": -0.352999985218048, "csq_phastcons100way_vertebrate": 0.0, + "discovery_interpretations": [ + "1e1ea605-b3e2-4abb-8677-d647042e4fc4", + "e8046b1b-22a4-40fa-8558-4702dae6d822" + ], + "interpretations": [ + "2f0557f6-7284-4a5f-976d-0f11f24567fd", + "de5e1c12-4c88-4a3e-a306-d37a12defa6b" + ], + "variant_notes": [ + "0791a219-856d-41b8-bc7a-7dbbc973f848", + "62759f07-326a-4e94-8dd0-366741292d07" + ], "uuid": "4af362a4-4f7f-4ad9-9c80-bf1f94bc143f" }, { diff --git a/src/encoded/tests/data/workbook-inserts/variant_sample.json b/src/encoded/tests/data/workbook-inserts/variant_sample.json index b0bf21c61b..61375613fe 100644 --- a/src/encoded/tests/data/workbook-inserts/variant_sample.json +++ b/src/encoded/tests/data/workbook-inserts/variant_sample.json @@ -56,9 +56,29 @@ "Heterozygous" ], "sample_id": "SAM10254-S1" + }, + { + "role": "mother", + "labels": [ + "Homozygous alternate" + ] + }, + { + "role": "father", + "labels": [ + "Heterozygous" + ] } ], - "inheritance_modes": [], + "novoPP": 0.5, + "cmphet": [ + {"comhet_mate_variant": "chr11:1016779G>A"} + ], + "inheritance_modes": ["paternal dominant"], + "interpretation": "ab5e1c89-4c88-4a3e-a306-d37a12defd8b", + "discovery_interpretation": "1e1ea605-b3e2-4abb-8677-d647042e4fc4", + "variant_notes": "62759f07-326a-4e94-8dd0-366741292d07", + "gene_notes": "62759f07-326a-4e94-8dd0-366741292d07", "uuid": "e43be20e-5dda-4db2-bb23-8c103323fc0f", "date_created": "2021-09-20T00:00:00.000000+00:00" }, diff --git a/src/encoded/tests/test_batch_download.py b/src/encoded/tests/test_batch_download.py new file mode 100644 index 0000000000..354350a24b --- /dev/null +++ b/src/encoded/tests/test_batch_download.py @@ -0,0 +1,935 @@ +import json +from contextlib import contextmanager +from copy import deepcopy +from unittest import mock +from typing import Any, Iterable, Iterator, List, Optional, Union + +import pytest +from pyramid.httpexceptions import HTTPBadRequest +from webtest.app import TestApp +from webtest.response import TestResponse + +from .utils import patch_context +from .. import ( + batch_download as batch_download_module, + batch_download_utils as batch_download_utils_module, +) +from ..batch_download import ( + CASE_SPREADSHEET_URL, + VARIANT_SAMPLE_SPREADSHEET_URL, + CaseSpreadsheet, + VariantSampleSpreadsheet, + validate_spreadsheet_file_format, + validate_spreadsheet_search_parameters, +) +from ..batch_download_utils import ( + OrderedSpreadsheetColumn, + SpreadsheetColumn, + SpreadsheetCreationError, + SpreadsheetRequest, +) +from ..item_models import Note, VariantSample +from ..util import APPLICATION_FORM_ENCODED_MIME_TYPE, JsonObject + + +EXPECTED_CASE_SPREADSHEET_COLUMNS = [ + ("# Case identifier", "Case ID", "CASE10254-S1-C1"), + ("Unique database identifier", "UUID", "165ad0fb-7acb-469e-bc1e-eb2fc6f94c82"), + ("Individual identifier", "Individual ID", "BRCA proband"), + ("Sex of associated individual", "Individual sex", "M"), + ("Whether case is for a proband", "Proband case", "False"), + ("Family identifier", "Family ID", "BRCA-001"), + ("Analysis type", "Analysis type", "WES-Group"), + ("Primary sample identifier", "Sample ID", "BRCA_proband_sample"), + ("Primary sample sequencing type", "Sequencing", "WES"), + ("Overall QC flag", "QC flag", "fail"), + ("Completed QC steps", "Completed QC", "BAM, SNV, SV"), + ( + "QC steps with warning flags", + "QC warnings", + "predicted_sex, heterozygosity_ratio", + ), + ( + "QC steps with failure flags", + "QC failures", + "coverage, transition_transversion_ratio", + ), +] +EXPECTED_VARIANT_SAMPLE_FILTERS_ROW = [ + "#", + "Filters Selected:", + "", + ( + "( associated_genotype_labels.proband_genotype_label = Heterozygous" + " & associated_genelists = Breast Cancer (28)" + " & variant.genes.genes_most_severe_consequence.impact = [ MODERATE | HIGH ] )" + " OR ( GQ.from = 60 & GQ.to = 99" + " & associated_genotype_labels.proband_genotype_label = Heterozygous" + " & associated_genelists = Familial Cancer (148)" + " & variant.csq_clinvar_clnsig = [ Uncertain_significance | Pathogenic ]" + " & variant.csq_gnomadg_af.from = 0 & variant.csq_gnomadg_af.to = 0.001" + " & variant.genes.genes_most_severe_consequence.impact = [ MODERATE | HIGH ] )" + " OR ( variant.csq_gnomade2_af.from = 0 & variant.csq_gnomade2_af.to = 0.001" + " & variant.csq_gnomadg_af.from = 0 & variant.csq_gnomadg_af.to = 0.001 )" + ), +] +EXPECTED_VARIANT_SAMPLE_SPACER_ROW = [ + "## -------------------------------------------------------" +] +EXPECTED_VARIANT_SAMPLE_SPREADSHEET_COLUMNS = [ + ( + "# URL path to the variant", + "ID", + "/variant-samples/e43be20e-5dda-4db2-bb23-8c103323fc0f/", + "/variant-samples/d62836a0-2de3-4970-bf14-eba3ca758a82/", + ), + ("Chromosome (hg38)", "Chrom (hg38)", "1", "16"), + ("Start position (hg38)", "Pos (hg38)", "2030666", "23630011"), + ("Chromosome (hg19)", "Chrom (hg19)", "1", "16"), + ("Start position (hg19)", "Pos (hg19)", "1962105", "23641332"), + ("Reference Nucleotide", "Ref", "G", "C"), + ("Alternate Nucleotide", "Alt", "A", "CTTA"), + ("Proband Genotype", "Proband genotype", "Heterozygous", "Heterozygous"), + ("Mother Genotype", "Mother genotype", "Homozygous alternate", ""), + ("Father Genotype", "Father genotype", "Heterozygous", ""), + ( + "HGVS genomic nomenclature", + "HGVSG", + "NC_000001.11:g.2030666G>A", + "NC_000016.10:g.23630011_23630012insTTA", + ), + ( + "HGVS cPos nomenclature", + "HGVSC", + "ENST00000378585.7:c.*384G>A", + "ENST00000261584.9:c.2142_2143insTAA", + ), + ( + "HGVS pPos nomenclature", + "HGVSP", + "ENST00000378585.7:p.V>A", + "ENSP00000261584.4:p.Asp714_Asp715insTer", + ), + ("dbSNP ID of variant", "dbSNP ID", "rs142404438", "rs876658855"), + ("Gene symbol(s)", "Genes", "GABRD", "PALB2"), + ( + "Ensembl ID of canonical transcript of gene variant is in", + "Canonical transcript ID", + "ENST00000378585", + "ENST00000261584", + ), + ( + "Number of exon or intron variant is located in canonical transcript, out of total", + "Canonical transcript location", + "Exon 9/9 (3' UTR)", + "Exon 5/13", + ), + ( + "Coding effect of variant in canonical transcript", + "Canonical transcript coding effect", + "3_prime_UTR_variant", + "stop_gained, inframe_insertion", + ), + ( + "Ensembl ID of transcript with worst annotation for variant", + "Most severe transcript ID", + "ENST00000378585", + "ENST00000261584", + ), + ( + "Number of exon or intron variant is located in most severe transcript, out of total", + "Most severe transcript location", + "Exon 9/9 (3' UTR)", + "Exon 5/13", + ), + ( + "Coding effect of variant in most severe transcript", + "Most severe transcript coding effect", + "3_prime_UTR_variant", + "stop_gained, inframe_insertion", + ), + ("Inheritance Modes of variant", "Inheritance modes", "paternal dominant", ""), + ("Novocaller Posterior Probability", "NovoPP", "0.5", ""), + ( + "Variant ID of mate, if variant is part of a compound heterozygous group", + "Cmphet mate", + "chr11:1016779G>A", + "", + ), + ("Variant call quality score", "Variant Quality", "688.12", "261.94"), + ("Genotype call quality score", "Genotype Quality", "99", "24"), + ("Strand bias estimated using Fisher's exact test", "Strand Bias", "6.249", "0"), + ("Number of reads with variant allele", "Allele Depth", "10", "12"), + ("Total number of reads at position", "Read Depth", "20", "30"), + ("Clinvar ID of variant", "clinvar ID", "89991", "230941"), + ( + "Total allele frequency in gnomad v3 (genomes)", + "gnomADv3 total AF", + "0.000197078", + "", + ), + ( + "Max. allele frequency in gnomad v3 (genomes)", + "gnomADv3 popmax AF", + "0.00500192", + "", + ), + ( + "Population with max. allele frequency in gnomad v3 (genomes)", + "gnomADv3 popmax population", + "East Asian", + "", + ), + ( + "Total allele frequency in gnomad v2 (exomes)", + "gnomADv2 exome total AF", + "0.0002", + "", + ), + ( + "Max. allele frequency in gnomad v2 (exomes)", + "gnomADv2 exome popmax AF", + "0.3", + "", + ), + ( + "Population with max. allele frequency in gnomad v2 (exomes)", + "gnomADv2 exome popmax population", + "East Asian", + "", + ), + ("GERP++ score", "GERP++", "5.02", ""), + ("CADD score", "CADD", "0.613", ""), + ("phyloP (30 Mammals) score", "phyloP-30M", "-3.49499988555908", ""), + ("phyloP (100 Vertebrates) score", "phyloP-100V", "-0.352999985218048", ""), + ("phastCons (100 Vertebrates) score", "phastCons-100V", "0.0", ""), + ("SIFT prediction", "SIFT", "T", ""), + ("PolyPhen2 prediction", "PolyPhen2", "D", ""), + ("Primate AI prediction", "PrimateAI", "D", ""), + ("REVEL score", "REVEL", "0.013", ""), + ("SpliceAI score", "SpliceAI", "0", "0"), + ( + "Loss-of-function observed/expected upper bound fraction", + "LOEUF", + "0.245", + "1.006", + ), + ( + "Estimates of heterozygous selection (source: Cassa et al 2017 Nat Genet doi:10.1038/ng.3831)", + "S-het", + "0.111583011", + "0.011004153", + ), + ( + "ACMG classification for variant in this case", + "ACMG classification (current)", + "Uncertain significance", + "", + ), + ("ACMG rules invoked for variant in this case", "ACMG rules (current)", "PM1", ""), + ( + "Clinical interpretation notes written for this case", + "Clinical interpretation notes (current)", + "This variant has been reported in the ClinVar database as Unknown Significance.", + "", + ), + ( + "Gene candidacy level selected for this case", + "Gene candidacy (current)", + "Strong candidate", + "", + ), + ( + "Variant candidacy level selected for this case", + "Variant candidacy (current)", + "Moderate candidate", + "", + ), + ( + "Gene/variant discovery notes written for this case", + "Discovery notes (current)", + "This gene is a real discovery!", + "", + ), + ( + "Additional notes on variant written for this case", + "Variant notes (current)", + "What a note", + "", + ), + ( + "Additional notes on gene written for this case", + "Gene notes (current)", + "What a note", + "", + ), + ( + "ACMG classification for variant in previous cases", + "ACMG classification (previous)", + "Pathogenic", + "", + ), + ( + "ACMG rules invoked for variant in previous cases", + "ACMG rules (previous)", + "PS1", + "", + ), + ( + "Clinical interpretation notes written for previous cases", + "Clinical interpretation (previous)", + "This variant has been reported in the ClinVar database as Pathogenic.", + "", + ), + ( + "Gene candidacy level selected for previous cases", + "Gene candidacy (previous)", + "Moderate candidate", + "", + ), + ( + "Variant candidacy level selected for previous cases", + "Variant candidacy (previous)", + "Weak candidate", + "", + ), + ( + "Gene/variant discovery notes written for previous cases", + "Discovery notes (previous)", + "This gene is not a real discovery...", + "", + ), + ( + "Additional notes on variant written for previous cases", + "Variant notes (previous)", + "What a poor note", + "", + ), + ( + "Additional notes on gene written for previous cases", + "Gene notes (previous)", + "What a poor note", + "", + ), +] +SOME_TITLE = "title" +SOME_DESCRIPTION = "description" +SOME_PROPERTY_COLUMN_TUPLE = (SOME_TITLE, SOME_DESCRIPTION, "fu") +SOME_CALLABLE_COLUMN_TUPLE = (SOME_TITLE, SOME_DESCRIPTION, print) +SOME_COLUMN_TUPLES = [SOME_PROPERTY_COLUMN_TUPLE, SOME_CALLABLE_COLUMN_TUPLE] +SOME_SPREADSHEET_COLUMNS = [SpreadsheetColumn(*column) for column in SOME_COLUMN_TUPLES] + + +@contextmanager +def patch_variant_sample_spreadsheet_column_tuples( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSampleSpreadsheet._get_column_tuples, + **kwargs, + ) as mock_get_column_tuples: + yield mock_get_column_tuples + + +@contextmanager +def patch_variant_sample_spreadsheet_columns(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSampleSpreadsheet._spreadsheet_columns, + **kwargs, + ) as mock_get_columns: + yield mock_get_columns + + +@contextmanager +def patch_variant_sample_spreadsheet_evaluate_item( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSampleSpreadsheet._evaluate_item_with_column, + **kwargs, + ) as mock_evaluate_item: + yield mock_evaluate_item + + +@contextmanager +def patch_variant_sample_spreadsheet_get_header_lines( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSampleSpreadsheet._get_available_header_lines, + **kwargs, + ) as mock_get_available_header_lines: + yield mock_get_available_header_lines + + +@contextmanager +def patch_variant_sample_spreadsheet_add_embeds(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSampleSpreadsheet._add_embeds, + **kwargs, + ) as mock_add_embeds: + yield mock_add_embeds + + +@contextmanager +def patch_variant_sample_spreadsheet_get_note_by_subrequest( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSampleSpreadsheet._get_note_by_subrequest, + **kwargs, + ) as mock_get_note_by_subrequest: + yield mock_get_note_by_subrequest + + +@contextmanager +def patch_variant_sample_spreadsheet_get_note_properties( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSampleSpreadsheet._get_note_properties, + **kwargs, + ) as mock_get_note_properties: + yield mock_get_note_properties + + +@contextmanager +def patch_variant_sample(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.VariantSample, + **kwargs, + ) as mock_variant_sample: + yield mock_variant_sample + + +@contextmanager +def patch_get_values_for_field(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_module.get_values_for_field, **kwargs + ) as mock_get_values_for_field: + yield mock_get_values_for_field + + +@contextmanager +def patch_spreadsheet_request_case_accession(**kwargs: Any) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.SpreadsheetRequest.get_case_accession, **kwargs + ) as mock_get_case_accession: + yield mock_get_case_accession + + +@contextmanager +def patch_spreadsheet_request_case_title(**kwargs: Any) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.SpreadsheetRequest.get_case_title, **kwargs + ) as mock_get_case_title: + yield mock_get_case_title + + +@contextmanager +def patch_spreadsheet_request_file_format(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.SpreadsheetRequest.get_file_format, **kwargs + ) as mock_get_file_format: + yield mock_get_file_format + + +@contextmanager +def patch_spreadsheet_request_compound_search(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.SpreadsheetRequest.get_compound_search, **kwargs + ) as mock_get_file_format: + yield mock_get_file_format + + +def mock_spreadsheet_request() -> mock.MagicMock: + return mock.create_autospec(SpreadsheetRequest, instance=True) + + +def mock_variant_sample() -> mock.MagicMock: + return mock.create_autospec(VariantSample, instance=True) + + +def assert_column_tuples(column_tuples: Iterable[OrderedSpreadsheetColumn]) -> None: + # Should be an easier way to do this with the annotation type directly... + assert isinstance(column_tuples, Iterable) + for column_tuple in column_tuples: + assert len(column_tuple) == 3 + (title, description, evaluator) = column_tuple + assert isinstance(title, str) + assert isinstance(description, str) + assert isinstance(evaluator, str) or callable(evaluator) + + +def parse_spreadsheet_response(response: TestResponse) -> List[List[str]]: + result = [] + for row in response.body.decode().split("\n"): + if not row: + continue + formatted_row = [] + cells = row.strip("\r").split("\t") + for cell in cells: + formatted_cell = cell.strip('"') + formatted_row.append(formatted_cell) + result.append(formatted_row) + return result + + +@pytest.mark.parametrize( + "file_format,expected_exception", + [ + ("tsv", False), + ("csv", False), + ("foo", True), + ], +) +def test_validate_spreadsheet_file_format( + file_format: str, expected_exception: bool +) -> None: + with patch_spreadsheet_request_file_format(return_value=file_format): + if expected_exception: + with pytest.raises(HTTPBadRequest): + validate_spreadsheet_file_format(None, None) + else: + validate_spreadsheet_file_format(None, None) + + +@pytest.mark.parametrize( + "compound_search,expected_exception", + [ + ("", True), + ("foo", False), + ], +) +def test_validate_spreadsheet_search_parameters( + compound_search: str, expected_exception: bool +) -> None: + with patch_spreadsheet_request_compound_search(return_value=compound_search): + if expected_exception: + with pytest.raises(HTTPBadRequest): + validate_spreadsheet_search_parameters(None, None) + else: + validate_spreadsheet_search_parameters(None, None) + + +@pytest.mark.workbook +def test_variant_sample_spreadsheet_download( + html_es_testapp: TestApp, es_testapp: TestApp, workbook: None +) -> None: + """Integrated test of variant sample search spreadsheet. + + Ensure all fields present on at least one VariantSample included in + the spreadsheet. + + Test with both a JSON and an HTML form POST; the latter is used by + front-end in production. + """ + compound_filterset = { + "search_type": "VariantSample", + "global_flags": ( + "CALL_INFO=SAM10254-S1&file=GAPFI3EBH4X2" + "&additional_facet=proband_only_inheritance_modes&sort=date_created" + ), + "intersect": False, + "filter_blocks": [ + { + "query": ( + "associated_genotype_labels.proband_genotype_label=Heterozygous" + "&associated_genelists=Breast+Cancer+%2828%29" + "&variant.genes.genes_most_severe_consequence.impact=MODERATE" + "&variant.genes.genes_most_severe_consequence.impact=HIGH" + ), + "flags_applied": [], + }, + { + "query": ( + "GQ.from=60&GQ.to=99" + "&associated_genotype_labels.proband_genotype_label=Heterozygous" + "&associated_genelists=Familial+Cancer+%28148%29" + "&variant.csq_clinvar_clnsig=Uncertain_significance" + "&variant.csq_clinvar_clnsig=Pathogenic" + "&variant.csq_gnomadg_af.from=0&variant.csq_gnomadg_af.to=0.001" + "&variant.genes.genes_most_severe_consequence.impact=MODERATE" + "&variant.genes.genes_most_severe_consequence.impact=HIGH" + ), + "flags_applied": [], + }, + { + "query": ( + "variant.csq_gnomade2_af.from=0&variant.csq_gnomade2_af.to=0.001" + "&variant.csq_gnomadg_af.from=0&variant.csq_gnomadg_af.to=0.001" + ), + "flags_applied": [], + }, + ], + } + post_body = {"compound_search_request": json.dumps(compound_filterset)} + json_post_response = es_testapp.post_json( + VARIANT_SAMPLE_SPREADSHEET_URL, + post_body, + status=200, + ) + json_post_rows = parse_spreadsheet_response(json_post_response) + + form_post_response = html_es_testapp.post( + VARIANT_SAMPLE_SPREADSHEET_URL, + post_body, + content_type=APPLICATION_FORM_ENCODED_MIME_TYPE, + status=200, + ) + form_post_rows = parse_spreadsheet_response(form_post_response) + + assert json_post_rows == form_post_rows + + rows = json_post_rows + assert len(rows) == 6 + + filters_row = rows[0] + assert filters_row == EXPECTED_VARIANT_SAMPLE_FILTERS_ROW + + spacer_row = rows[1] + assert spacer_row == EXPECTED_VARIANT_SAMPLE_SPACER_ROW + + variant_sample_rows = rows[2:] + columns = list(zip(*variant_sample_rows)) + assert columns == EXPECTED_VARIANT_SAMPLE_SPREADSHEET_COLUMNS + + +@pytest.mark.workbook +def test_case_search_spreadsheet( + html_es_testapp: TestApp, es_testapp: TestApp, workbook: None +) -> None: + """Integrated test of case search spreadsheet. + + Ensure all fields present on at least one Case included in the + spreadsheet. + + Test with both a JSON and an HTML form POST; the latter is used by + front-end in production. + """ + case_search_compound_filterset = { + "search_type": "Case", + "global_flags": "case_id=CASE10254-S1-C1", + } + post_body = {"compound_search_request": json.dumps(case_search_compound_filterset)} + json_post_response = es_testapp.post_json( + CASE_SPREADSHEET_URL, + post_body, + status=200, + ) + json_post_rows = parse_spreadsheet_response(json_post_response) + + form_post_response = html_es_testapp.post( + CASE_SPREADSHEET_URL, + post_body, + content_type=APPLICATION_FORM_ENCODED_MIME_TYPE, + status=200, + ) + form_post_rows = parse_spreadsheet_response(form_post_response) + + assert json_post_rows == form_post_rows + + rows = json_post_rows + assert len(rows) == 3 + + columns = list(zip(*rows)) + assert columns == EXPECTED_CASE_SPREADSHEET_COLUMNS + + +class TestVariantSampleSpreadsheet: + + SOME_GENE_NOTE_ATID = "/foo/bar/" + SOME_GENE_NOTE = {"@id": SOME_GENE_NOTE_ATID, "foo": "bar"} + SOME_UPDATED_GENE_NOTE = {**SOME_GENE_NOTE, "fu": "bur"} + SOME_VARIANT_SAMPLE_PROPERTIES = { + "variant": {"something": "else"}, + "gene_notes": [SOME_GENE_NOTE], + } + SOME_VARIANT_SAMPLE_PROPERTIES_WITH_UPDATED_NOTE = { + **SOME_VARIANT_SAMPLE_PROPERTIES, + "gene_notes": [SOME_UPDATED_GENE_NOTE], + } + SOME_VARIANT_SAMPLE = VariantSample(SOME_VARIANT_SAMPLE_PROPERTIES) + + def get_variant_sample_spreadsheet( + self, + embed_additional_items: bool = False, + spreadsheet_request: Optional[mock.Mock] = None, + ) -> VariantSampleSpreadsheet: + return VariantSampleSpreadsheet( + [self.SOME_VARIANT_SAMPLE_PROPERTIES], + embed_additional_items=embed_additional_items, + spreadsheet_request=spreadsheet_request, + ) + + @pytest.mark.parametrize( + "header_lines,expected", + [ + ([], []), + ([["foo"]], [["foo"], VariantSampleSpreadsheet.HEADER_SPACER_LINE]), + ], + ) + def test_get_headers( + self, header_lines: List[List[str]], expected: List[List[str]] + ) -> None: + with patch_variant_sample_spreadsheet_get_header_lines( + return_value=header_lines + ): + spreadsheet = self.get_variant_sample_spreadsheet() + result = spreadsheet._get_headers() + assert result == expected + + def test_get_available_header_lines(self) -> None: + case_accession = case_title = "foo" + compound_search = {"filter_blocks": [{"search_type": "item"}]} + expected_filters = "" + with patch_spreadsheet_request_case_accession(return_value=case_accession): + with patch_spreadsheet_request_case_title(return_value=case_title): + with patch_spreadsheet_request_compound_search( + return_value=compound_search + ): + spreadsheet_request = SpreadsheetRequest(None) + spreadsheet = self.get_variant_sample_spreadsheet( + spreadsheet_request=spreadsheet_request + ) + result = spreadsheet._get_available_header_lines() + assert result == [ + ["#", "Case Accession:", "", case_accession], + ["#", "Case Title:", "", case_title], + ["#", "Filters Selected:", "", expected_filters], + ] + + def test_get_column_titles(self) -> None: + with patch_variant_sample_spreadsheet_column_tuples( + return_value=SOME_COLUMN_TUPLES + ): + spreadsheet = self.get_variant_sample_spreadsheet() + result = spreadsheet._get_column_titles() + assert list(result) == [SOME_TITLE] * 2 + + def test_get_column_descriptions(self) -> None: + with patch_variant_sample_spreadsheet_column_tuples( + return_value=SOME_COLUMN_TUPLES + ): + spreadsheet = self.get_variant_sample_spreadsheet() + result = spreadsheet._get_column_descriptions() + assert list(result) == [f"# {SOME_DESCRIPTION}", SOME_DESCRIPTION] + + def test_get_row_for_item(self) -> None: + expected_result_count = len(SOME_COLUMN_TUPLES) + with patch_variant_sample_spreadsheet_columns( + return_value=SOME_SPREADSHEET_COLUMNS + ): + with patch_variant_sample_spreadsheet_evaluate_item() as mock_evaluate_item_with_column: + with patch_variant_sample_spreadsheet_add_embeds() as mock_add_embeds: + with patch_variant_sample(return_value=self.SOME_VARIANT_SAMPLE): + spreadsheet = self.get_variant_sample_spreadsheet() + result_generator = spreadsheet._get_row_for_item( + self.SOME_VARIANT_SAMPLE_PROPERTIES + ) + result = list(result_generator) + assert len(result) == expected_result_count + assert ( + len(mock_evaluate_item_with_column.call_args_list) + == expected_result_count + ) + mock_add_embeds.assert_called_once_with( + self.SOME_VARIANT_SAMPLE_PROPERTIES + ) + for column in SOME_SPREADSHEET_COLUMNS: + mock_evaluate_item_with_column.assert_any_call( + column, self.SOME_VARIANT_SAMPLE + ) + + @pytest.mark.parametrize( + "embed_additional_items,spreadsheet_request,expected", + [ + (False, None, SOME_VARIANT_SAMPLE_PROPERTIES), + (True, None, SOME_VARIANT_SAMPLE_PROPERTIES), + (False, mock_spreadsheet_request(), SOME_VARIANT_SAMPLE_PROPERTIES), + ( + True, + mock_spreadsheet_request(), + SOME_VARIANT_SAMPLE_PROPERTIES_WITH_UPDATED_NOTE, + ), + ], + ) + def test_add_embeds( + self, + embed_additional_items: bool, + spreadsheet_request: Union[None, mock.Mock], + expected: JsonObject, + ) -> None: + with patch_variant_sample_spreadsheet_get_note_by_subrequest( + return_value=self.SOME_UPDATED_GENE_NOTE + ) as mock_get_note: + spreadsheet = self.get_variant_sample_spreadsheet( + embed_additional_items=embed_additional_items, + spreadsheet_request=spreadsheet_request, + ) + item_to_evaluate = deepcopy(self.SOME_VARIANT_SAMPLE_PROPERTIES) + spreadsheet._add_embeds(item_to_evaluate) + assert item_to_evaluate == expected + if embed_additional_items and spreadsheet_request: + mock_get_note.assert_called_once_with(self.SOME_UPDATED_GENE_NOTE) + else: + mock_get_note.assert_not_called() + assert item_to_evaluate == expected + + @pytest.mark.parametrize( + ( + "is_property_evaluator,is_callable_evaluator,expected_exception," + "expected_field_for_item_call" + ), + [ + (True, False, False, SOME_VARIANT_SAMPLE_PROPERTIES), + (False, True, False, SOME_VARIANT_SAMPLE), + (True, True, False, SOME_VARIANT_SAMPLE_PROPERTIES), + (False, False, True, None), + ], + ) + def test_evaluate_item_with_column( + self, + is_property_evaluator: bool, + is_callable_evaluator: bool, + expected_exception: bool, + expected_field_for_item_call: Any, + ) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + column = self.make_mock_column(is_property_evaluator, is_callable_evaluator) + if expected_exception: + with pytest.raises(SpreadsheetCreationError): + spreadsheet._evaluate_item_with_column(column, self.SOME_VARIANT_SAMPLE) + else: + result = spreadsheet._evaluate_item_with_column( + column, self.SOME_VARIANT_SAMPLE + ) + assert result == column.get_field_for_item.return_value + column.get_field_for_item.assert_called_once_with( + expected_field_for_item_call + ) + + def make_mock_column( + self, is_property_evaluator: bool, is_callable_evaluator: bool + ) -> mock.MagicMock: + column = mock.create_autospec(SpreadsheetColumn, instance=True) + column.is_property_evaluator.return_value = is_property_evaluator + column.is_callable_evaluator.return_value = is_callable_evaluator + return column + + def test_get_column_tuples(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + result = spreadsheet._get_column_tuples() + assert_column_tuples(result) + + def test_get_canonical_transcript_feature(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_canonical_transcript_feature(variant_sample) + assert result == variant_sample.get_canonical_transcript_feature.return_value + + def test_get_canonical_transcript_location(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_canonical_transcript_location(variant_sample) + assert result == variant_sample.get_canonical_transcript_location.return_value + + def test_get_canonical_transcript_consequence_names(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_canonical_transcript_consequence_names(variant_sample) + assert ( + result + == variant_sample.get_canonical_transcript_consequence_names.return_value + ) + + def test_get_most_severe_transcript_feature(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_most_severe_transcript_feature(variant_sample) + assert result == variant_sample.get_most_severe_transcript_feature.return_value + + def test_get_most_severe_transcript_location(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_most_severe_transcript_location(variant_sample) + assert result == variant_sample.get_most_severe_transcript_location.return_value + + def test_get_most_severe_transcript_consequence_names(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_most_severe_transcript_consequence_names( + variant_sample + ) + assert ( + result + == variant_sample.get_most_severe_transcript_consequence_names.return_value + ) + + def test_get_gnomad_v3_popmax_population(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_gnomad_v3_popmax_population(variant_sample) + assert result == variant_sample.get_gnomad_v3_popmax_population.return_value + + def test_get_gnomad_v2_popmax_population(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + variant_sample = mock_variant_sample() + result = spreadsheet._get_gnomad_v2_popmax_population(variant_sample) + assert result == variant_sample.get_gnomad_v2_popmax_population.return_value + + def test_get_note_of_same_project(self) -> None: + spreadsheet = self.get_variant_sample_spreadsheet() + property_location = "foo" + property_to_retrieve = "bar" + with patch_variant_sample_spreadsheet_get_note_properties() as mock_get_note_properties: + result = spreadsheet._get_note_of_same_project( + property_location, property_to_retrieve + ) + assert callable(result) + assert result.func == mock_get_note_properties + assert not result.args + assert result.keywords == { + "note_property_location": property_location, + "note_property_to_retrieve": property_to_retrieve, + } + + @pytest.mark.parametrize( + "note_found,expected", + [ + (False, ""), + (True, SOME_GENE_NOTE_ATID), + ], + ) + def test_get_note_properties(self, note_found: bool, expected: bool) -> None: + note_property_location = "foo" + note_property_to_retrieve = "@id" # Present in SOME_GENE_NOTE + variant_sample = mock_variant_sample() + if note_found: + return_value = Note(self.SOME_GENE_NOTE) + else: + return_value = None + get_note = variant_sample.get_most_recent_note_of_same_project_from_property + get_note.return_value = return_value + spreadsheet = self.get_variant_sample_spreadsheet() + result = spreadsheet._get_note_properties( + variant_sample, + note_property_location=note_property_location, + note_property_to_retrieve=note_property_to_retrieve, + ) + get_note.assert_called_once_with(note_property_location) + assert result == expected + + +class TestCaseSpreadsheet: + @pytest.mark.parametrize( + "to_evaluate,expected", + [ + ({}, CaseSpreadsheet.NO_FLAG_DEFAULT), + ({"quality_control_flags": {}}, CaseSpreadsheet.NO_FLAG_DEFAULT), + ({"quality_control_flags": {"flag": "pass"}}, "pass"), + ], + ) + def test_get_qc_flag(self, to_evaluate: JsonObject, expected: str) -> None: + result = CaseSpreadsheet._get_qc_flag(to_evaluate) + assert result == expected diff --git a/src/encoded/tests/test_batch_download_utils.py b/src/encoded/tests/test_batch_download_utils.py new file mode 100644 index 0000000000..ba3d164ad2 --- /dev/null +++ b/src/encoded/tests/test_batch_download_utils.py @@ -0,0 +1,483 @@ +import json +from contextlib import contextmanager +from typing import ( + Any, + Callable, + Generator, + Iterator, + List, + Optional, + Sequence, + Union, +) +from unittest import mock + +import pytest +from pyramid.request import Request +from pyramid.response import Response + + +from .utils import patch_context +from .. import batch_download_utils as batch_download_utils_module +from ..batch_download_utils import ( + DEFAULT_FILE_FORMAT, + OrderedSpreadsheetColumn, + get_values_for_field, + FilterSetSearch, + SpreadsheetColumn, + SpreadsheetCreationError, + SpreadsheetFromColumnTuples, + SpreadsheetGenerator, + SpreadsheetRequest, + SpreadsheetTemplate, +) +from ..util import JsonObject + + +SOME_REQUEST = "foo" +SOME_CONTEXT = "bar" + + +@contextmanager +def patch_compound_search_builder_extract_filter_set( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.CompoundSearchBuilder.extract_filter_set_from_search_body, + **kwargs, + ) as mocked_item: + yield mocked_item + + +@contextmanager +def patch_compound_search_builder_execute_filter_set( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.CompoundSearchBuilder.execute_filter_set, **kwargs + ) as mocked_item: + yield mocked_item + + +@contextmanager +def patch_spreadsheet_request_parameters(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.SpreadsheetRequest.parameters, **kwargs + ) as mocked_item: + yield mocked_item + + +@contextmanager +def patch_filter_set_search_get_filter_set(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.FilterSetSearch._get_filter_set, **kwargs + ) as mocked_item: + yield mocked_item + + +@contextmanager +def patch_get_values_for_field(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + batch_download_utils_module.get_values_for_field, **kwargs + ) as mocked_item: + yield mocked_item + + +def mock_request() -> mock.MagicMock: + return mock.create_autospec(Request) + + +@pytest.mark.parametrize( + "item,field,remove_duplicates,expected", + [ + ({"foo": "bar"}, "bar", True, ""), + ({"foo": "bar"}, "foo", True, "bar"), + ({"foo": 1}, "foo", True, "1"), + ({"foo": ["bar"]}, "foo", True, "bar"), + ({"foo": ["bar", "bar"]}, "foo", True, "bar"), + ({"foo": ["bar", "bar"]}, "foo", False, "bar, bar"), + ({"foo": [{"fu": "bar"}]}, "foo", True, "{'fu': 'bar'}"), + ({"foo": [{"fu": "bar"}]}, "foo.fu", True, "bar"), + ({"foo": [{"fu": "bar"}, {"fu": "bur"}]}, "foo.fu", True, "bar, bur"), + ], +) +def test_get_values_for_field( + item: JsonObject, field: str, remove_duplicates: bool, expected: str +) -> None: + result = get_values_for_field(item, field, remove_duplicates=remove_duplicates) + assert result == expected + + +def evaluate_item(item: JsonObject) -> bool: + return True + + +class TestSpreadsheetColumn: + + SOME_TITLE = "column_a" + SOME_DESCRIPTION = "A lovely little column" + SOME_FIELD_TO_EVALUATE = "foo" + SOME_FIELD_VALUE = "bar" + SOME_ITEM_TO_EVALUATE = {SOME_FIELD_TO_EVALUATE: SOME_FIELD_VALUE} + SOME_CALLABLE_EVALUATOR = evaluate_item + + def get_spreadsheet_column( + self, evaluator: Union[str, Callable] = SOME_FIELD_TO_EVALUATE + ) -> SpreadsheetColumn: + return SpreadsheetColumn(self.SOME_TITLE, self.SOME_DESCRIPTION, evaluator) + + @pytest.mark.parametrize( + "to_evaluate,evaluator,exception_expected,expected", + [ + (SOME_ITEM_TO_EVALUATE, SOME_FIELD_TO_EVALUATE, False, SOME_FIELD_VALUE), + ("not_a_dict", SOME_FIELD_TO_EVALUATE, True, None), + (SOME_ITEM_TO_EVALUATE, SOME_CALLABLE_EVALUATOR, False, True), + (SOME_ITEM_TO_EVALUATE, None, True, None), + ], + ) + def test_get_field_for_item( + self, to_evaluate: Any, evaluator: Any, exception_expected: bool, expected: Any + ) -> None: + spreadsheet_column = self.get_spreadsheet_column(evaluator=evaluator) + if exception_expected: + with pytest.raises(SpreadsheetCreationError): + spreadsheet_column.get_field_for_item(to_evaluate) + else: + result = spreadsheet_column.get_field_for_item(to_evaluate) + assert result == expected + + def test_get_field_from_item(self) -> None: + with patch_get_values_for_field( + return_value=self.SOME_FIELD_VALUE + ) as mock_get_values_for_field: + spreadsheet_column = self.get_spreadsheet_column() + result = spreadsheet_column._get_field_from_item(self.SOME_ITEM_TO_EVALUATE) + assert result == self.SOME_FIELD_VALUE + mock_get_values_for_field.assert_called_once_with( + self.SOME_ITEM_TO_EVALUATE, self.SOME_FIELD_TO_EVALUATE + ) + + @pytest.mark.parametrize( + "evaluator,expected", + [ + (SOME_FIELD_TO_EVALUATE, True), + (SOME_CALLABLE_EVALUATOR, False), + ], + ) + def test_is_property_evaluator( + self, evaluator: Union[str, Callable], expected: bool + ) -> None: + spreadsheet_column = self.get_spreadsheet_column(evaluator=evaluator) + assert spreadsheet_column.is_property_evaluator() == expected + + @pytest.mark.parametrize( + "evaluator,expected", + [ + (SOME_FIELD_TO_EVALUATE, False), + (SOME_CALLABLE_EVALUATOR, True), + ], + ) + def test_is_callable_evaluator( + self, evaluator: Union[str, Callable], expected: bool + ) -> None: + spreadsheet_column = self.get_spreadsheet_column(evaluator=evaluator) + assert spreadsheet_column.is_callable_evaluator() == expected + + +class TestSpreadsheetTemplate: + + SOME_ITEM_TO_EVALUATE = {"foo": "bar"} + ITEMS_TO_EVALUATE = [SOME_ITEM_TO_EVALUATE] + + class Spreadsheet(SpreadsheetTemplate): + + SOME_HEADER = ["foo", "bar"] + SOME_HEADERS = [SOME_HEADER] + SOME_COLUMN_TITLES = ["title1", "title2"] + SOME_COLUMN_DESCRIPTIONS = ["description1", "description2"] + SOME_ITEM_ROW = ["item_name", "some_property"] + + def _get_headers(self) -> List[List[str]]: + return self.SOME_HEADERS + + def _get_column_titles(self) -> List[str]: + return self.SOME_COLUMN_TITLES + + def _get_column_descriptions(self) -> List[str]: + return self.SOME_COLUMN_DESCRIPTIONS + + def _get_row_for_item(self, item_to_evaluate: JsonObject) -> List[str]: + return self.SOME_ITEM_ROW + + def get_spreadsheet(self) -> Spreadsheet: + return self.Spreadsheet(self.ITEMS_TO_EVALUATE) + + def test_yield_rows(self) -> None: + spreadsheet = self.get_spreadsheet() + result = spreadsheet.yield_rows() + assert isinstance(result, Generator) + rows = list(result) + assert rows == [ + self.Spreadsheet.SOME_HEADER, + self.Spreadsheet.SOME_COLUMN_DESCRIPTIONS, + self.Spreadsheet.SOME_COLUMN_TITLES, + self.Spreadsheet.SOME_ITEM_ROW, + ] + + +class TestSpreadsheetFromColumnTuples: + class Spreadsheet(SpreadsheetFromColumnTuples): + @classmethod + def callable_evaluator(cls, to_evaluate: Any) -> str: + return "foo" + + SOME_TITLE_1 = "Some title" + SOME_TITLE_2 = "Another title" + SOME_DESCRIPTION_1 = "Some description" + SOME_DESCRIPTION_2 = "Another description" + SOME_COLUMN_TUPLES = [ + (SOME_TITLE_1, SOME_DESCRIPTION_1, "Some property"), + (SOME_TITLE_2, SOME_DESCRIPTION_2, callable_evaluator), + ] + + @classmethod + def _get_column_tuples(cls) -> List[OrderedSpreadsheetColumn]: + return cls.SOME_COLUMN_TUPLES + + def _get_headers(self) -> None: + return + + def _get_row_for_item(self) -> None: + return + + def get_spreadsheet(self) -> Spreadsheet: + return self.Spreadsheet([]) + + def test_get_spreadsheet_columns(self) -> None: + spreadsheet = self.get_spreadsheet() + result = spreadsheet.get_spreadsheet_columns() + assert len(result) == 2 + for item in result: + assert isinstance(item, SpreadsheetColumn) + + def test_get_column_titles(self) -> None: + spreadsheet = self.get_spreadsheet() + result = spreadsheet._get_column_titles() + assert result == [self.Spreadsheet.SOME_TITLE_1, self.Spreadsheet.SOME_TITLE_2] + + def test_get_column_descriptions(self) -> None: + spreadsheet = self.get_spreadsheet() + result = spreadsheet._get_column_descriptions() + assert result == [ + f"# {self.Spreadsheet.SOME_DESCRIPTION_1}", + self.Spreadsheet.SOME_DESCRIPTION_2, + ] + + +class TestSpreadsheetRequest: + + SOME_FILE_FORMAT = "something" + SOME_CASE_ACCESSION = "foo" + SOME_CASE_TITLE = "bar" + SOME_COMPOUND_SEARCH = {"search": "this"} + SOME_PARAMETERS = { + "file_format": SOME_FILE_FORMAT, + "case_accession": SOME_CASE_ACCESSION, + "case_title": SOME_CASE_TITLE, + "compound_search_request": SOME_COMPOUND_SEARCH, + } + + def get_spreadsheet_request(self) -> SpreadsheetRequest: + return SpreadsheetRequest(SOME_REQUEST) + + def test_parameters(self) -> None: + request = mock_request() + spreadsheet_request = SpreadsheetRequest(request) + assert spreadsheet_request.parameters == request.params + + request.params = None + assert spreadsheet_request.parameters == request.json + + @pytest.mark.parametrize( + "parameters,expected", + [ + ({}, DEFAULT_FILE_FORMAT), + (SOME_PARAMETERS, SOME_FILE_FORMAT), + ], + ) + def test_get_file_format(self, parameters: JsonObject, expected: str) -> None: + with patch_spreadsheet_request_parameters(return_value=parameters): + spreadsheet_request = self.get_spreadsheet_request() + assert spreadsheet_request.get_file_format() == expected + + @pytest.mark.parametrize( + "parameters,expected", + [ + ({}, ""), + (SOME_PARAMETERS, SOME_CASE_ACCESSION), + ], + ) + def test_get_case_accession(self, parameters: JsonObject, expected: str) -> None: + with patch_spreadsheet_request_parameters(return_value=parameters): + spreadsheet_request = self.get_spreadsheet_request() + assert spreadsheet_request.get_case_accession() == expected + + @pytest.mark.parametrize( + "parameters,expected", + [ + ({}, ""), + (SOME_PARAMETERS, SOME_CASE_TITLE), + ], + ) + def test_get_case_title(self, parameters: JsonObject, expected: str) -> None: + with patch_spreadsheet_request_parameters(return_value=parameters): + spreadsheet_request = self.get_spreadsheet_request() + assert spreadsheet_request.get_case_title() == expected + + @pytest.mark.parametrize( + "parameters,expected", + [ + ({}, {}), + (SOME_PARAMETERS, SOME_COMPOUND_SEARCH), + ( + {"compound_search_request": json.dumps(SOME_COMPOUND_SEARCH)}, + SOME_COMPOUND_SEARCH, + ), + ], + ) + def test_get_compound_search(self, parameters: JsonObject, expected: str) -> None: + with patch_spreadsheet_request_parameters(return_value=parameters): + spreadsheet_request = self.get_spreadsheet_request() + assert spreadsheet_request.get_compound_search() == expected + + +class TestFilterSetSearch: + + SOME_COMPOUND_SEARCH = {"foo": "bar"} + + def get_filter_set_search( + self, compound_search: Optional[JsonObject] = None + ) -> FilterSetSearch: + compound_search_to_use = compound_search or self.SOME_COMPOUND_SEARCH + return FilterSetSearch(SOME_CONTEXT, SOME_REQUEST, compound_search_to_use) + + def test_get_search_results(self) -> None: + filter_set_search = self.get_filter_set_search() + with patch_compound_search_builder_execute_filter_set() as mock_execute_filter_set: + with patch_filter_set_search_get_filter_set() as mock_get_filter_set: + filter_set_search.get_search_results() + mock_execute_filter_set.assert_called_once_with( + SOME_CONTEXT, + SOME_REQUEST, + mock_get_filter_set.return_value, + to="all", + global_flags=None, + intersect=False, + return_generator=True, + ) + + def test_get_filter_set(self) -> None: + filter_set_search = self.get_filter_set_search() + with patch_compound_search_builder_extract_filter_set() as mock_extract_filter_set: + filter_set_search._get_filter_set() + mock_extract_filter_set.assert_called_once_with( + SOME_REQUEST, self.SOME_COMPOUND_SEARCH + ) + + @pytest.mark.parametrize( + "compound_search,expected", + [ + ({}, None), + ({"global_flags": "something"}, "something"), + ], + ) + def test_get_global_flags( + self, compound_search: JsonObject, expected: Union[str, None] + ) -> None: + filter_set_search = self.get_filter_set_search(compound_search=compound_search) + assert filter_set_search._get_global_flags() == expected + + @pytest.mark.parametrize( + "compound_search,expected", + [ + ({}, False), + ({"intersect": False}, False), + ({"intersect": True}, True), + ], + ) + def test_is_intersect(self, compound_search: JsonObject, expected: bool) -> None: + filter_set_search = self.get_filter_set_search(compound_search=compound_search) + assert filter_set_search._is_intersect() == expected + + +class TestSpreadsheetGenerator: + + FILE_NAME = "foobar" + ROWS_TO_WRITE = [["a"], ["b"]] + BAD_FILE_FORMAT = "foo" + + def get_spreadsheet_generator( + self, + file_name: Optional[str] = None, + rows_to_write: Optional[Sequence[Sequence[str]]] = None, + file_format: Optional[str] = None, + ) -> SpreadsheetGenerator: + name = file_name or self.FILE_NAME + to_write = rows_to_write or self.ROWS_TO_WRITE + if file_format: + return SpreadsheetGenerator(name, to_write, file_format=file_format) + else: + return SpreadsheetGenerator(name, to_write) + + def test_get_streaming_response(self) -> None: + spreadsheet_generator = self.get_spreadsheet_generator() + result = spreadsheet_generator.get_streaming_response() + assert isinstance(result, Response) + assert isinstance(result.app_iter, Generator) + assert result.status_code == 200 + + @pytest.mark.parametrize( + "rows_to_write,expected", + [ + ([["a", "b"], ["c"]], [b'"a"\t"b"\r\n', b'"c"\r\n']), + ([["a", "b"], [], ["c"]], [b'"a"\t"b"\r\n', b'"c"\r\n']), + ([[5], [[1, 2]]], [b"5\r\n", b'"[1, 2]"\r\n']), + ([[{"a": "b"}]], [b"\"{'a': 'b'}\"\r\n"]), + ], + ) + def test_stream_spreadsheet( + self, rows_to_write: List[List[str]], expected: List[str] + ) -> None: + spreadsheet_generator = self.get_spreadsheet_generator( + rows_to_write=rows_to_write + ) + written_rows = spreadsheet_generator._stream_spreadsheet() + assert isinstance(written_rows, Generator) + assert list(written_rows) == expected + + @pytest.mark.parametrize( + "file_format,exception_expected,expected", + [ + ("tsv", False, "\t"), + ("foo", True, ""), + ], + ) + def test_get_delimiter( + self, file_format: str, exception_expected: bool, expected: str + ) -> None: + spreadsheet_generator = self.get_spreadsheet_generator(file_format=file_format) + if exception_expected: + with pytest.raises(SpreadsheetCreationError): + spreadsheet_generator._get_delimiter() + else: + assert spreadsheet_generator._get_delimiter() == expected + + def test_get_response_headers(self) -> None: + spreadsheet_generator = self.get_spreadsheet_generator() + assert spreadsheet_generator._get_response_headers() == { + "X-Accel-Buffering": "no", + "Content-Disposition": "attachment; filename=foobar.tsv", + "Content-Type": "text/tsv", + "Content-Description": "File Transfer", + "Cache-Control": "no-store", + } diff --git a/src/encoded/tests/test_item_models.py b/src/encoded/tests/test_item_models.py new file mode 100644 index 0000000000..8113672b27 --- /dev/null +++ b/src/encoded/tests/test_item_models.py @@ -0,0 +1,800 @@ +from contextlib import contextmanager +from typing import Any, Iterator, List, Optional, Union +from unittest import mock + +import pytest + +from .utils import patch_context +from .. import item_models as item_models_module +from ..item_models import ( + Item, + Transcript, + Note, + Variant, + VariantConsequence, + VariantSample, + VariantSampleList, +) +from ..util import JsonObject + + +@contextmanager +def patch_transcript_get_most_severe_consequence(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.Transcript._get_most_severe_consequence, **kwargs + ) as mock_get_most_severe: + yield mock_get_most_severe + + +@contextmanager +def patch_transcript_get_location_by_most_severe_consequence( + **kwargs, +) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.Transcript._get_location_by_most_severe_consequence, **kwargs + ) as mock_get_location_by_most_severe: + yield mock_get_location_by_most_severe + + +@contextmanager +def patch_variant_consequence_name(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.VariantConsequence._name, **kwargs + ) as mock_consequence: + yield mock_consequence + + +@contextmanager +def patch_variant_get_transcripts(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.Variant._get_transcripts, **kwargs + ) as mock_get_transcripts: + yield mock_get_transcripts + + +@contextmanager +def patch_variant_get_canonical_transcript(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.Variant._get_canonical_transcript, **kwargs + ) as mock_get_canonical_transcript: + yield mock_get_canonical_transcript + + +@contextmanager +def patch_variant_get_most_severe_transcript(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.Variant._get_most_severe_transcript, **kwargs + ) as mock_get_most_severe_transcript: + yield mock_get_most_severe_transcript + + +@contextmanager +def patch_variant_sample_variant(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.VariantSample._variant, **kwargs + ) as mock_variant: + yield mock_variant + + +@contextmanager +def patch_variant_sample_get_variant(**kwargs) -> Iterator[mock.MagicMock]: + with patch_context( + item_models_module.VariantSample._get_variant, **kwargs + ) as mock_get_variant: + yield mock_get_variant + + +def mock_variant_consequence( + is_upstream: bool = False, + is_downstream: bool = False, + is_three_prime_utr: bool = False, + is_five_prime_utr: bool = False, +) -> mock.MagicMock: + mock_variant_consequence = mock.create_autospec(VariantConsequence, instance=True) + mock_variant_consequence.is_upstream.return_value = is_upstream + mock_variant_consequence.is_downstream.return_value = is_downstream + mock_variant_consequence.is_three_prime_utr.return_value = is_three_prime_utr + mock_variant_consequence.is_five_prime_utr.return_value = is_five_prime_utr + return mock_variant_consequence + + +def mock_transcript( + canonical: bool = False, + most_severe: bool = False, +) -> mock.MagicMock: + mock_transcript = mock.create_autospec(Transcript, instance=True) + mock_transcript.is_canonical.return_value = canonical + mock_transcript.is_most_severe.return_value = most_severe + return mock_transcript + + +def mock_variant() -> mock.MagicMock: + return mock.create_autospec(Variant, instance=True) + + +class TestItem: + SOME_ATID = "/foo/bar" + SOME_PROJECT = "cgap-core" + SOME_ITEM_PROPERTIES = {"@id": SOME_ATID, "project": SOME_PROJECT} + SOME_ITEM = Item(SOME_ITEM_PROPERTIES) + SOME_EMPTY_ITEM = Item({}) + + @pytest.mark.parametrize( + "item,expected", + [ + (SOME_ITEM, SOME_ITEM_PROPERTIES), + (SOME_EMPTY_ITEM, {}), + ], + ) + def test_get_properties(self, item: Item, expected: JsonObject) -> None: + assert item.get_properties() == expected + + @pytest.mark.parametrize( + "item,expected", + [ + (SOME_ITEM, SOME_ATID), + (SOME_EMPTY_ITEM, ""), + ], + ) + def test_get_atid(self, item: Item, expected: JsonObject) -> None: + assert item.get_atid() == expected + + @pytest.mark.parametrize( + "item,expected", + [ + (SOME_ITEM, SOME_PROJECT), + (SOME_EMPTY_ITEM, ""), + ], + ) + def test_get_project(self, item: Item, expected: JsonObject) -> None: + assert item.get_project() == expected + + +class TestVariantConsequence: + + SOME_CONSEQUENCE_IMPACT = "high" + SOME_CONSEQUENCE_NAME = "gene_affected" + SOME_VARIANT_CONSEQUENCE_PROPERTIES = { + "impact": SOME_CONSEQUENCE_IMPACT, + "var_conseq_name": SOME_CONSEQUENCE_NAME, + } + SOME_VARIANT_CONSEQUENCE = VariantConsequence(SOME_VARIANT_CONSEQUENCE_PROPERTIES) + SOME_EMPTY_VARIANT_CONSEQUENCE = VariantConsequence({}) + + @pytest.mark.parametrize( + "variant_consequence,expected", + [ + (SOME_VARIANT_CONSEQUENCE, SOME_CONSEQUENCE_NAME), + (SOME_EMPTY_VARIANT_CONSEQUENCE, ""), + ], + ) + def test_get_name( + self, variant_consequence: VariantConsequence, expected: str + ) -> None: + result = variant_consequence.get_name() + assert result == expected + + @pytest.mark.parametrize( + "variant_consequence,expected", + [ + (SOME_VARIANT_CONSEQUENCE, SOME_CONSEQUENCE_IMPACT), + (SOME_EMPTY_VARIANT_CONSEQUENCE, ""), + ], + ) + def test_get_impact( + self, variant_consequence: VariantConsequence, expected: str + ) -> None: + result = variant_consequence.get_impact() + assert result == expected + + @pytest.mark.parametrize( + "consequence_name,expected", + [ + ("foo", False), + ("downstream_gene_variant", True), + ], + ) + def test_is_downstream(self, consequence_name: str, expected: bool) -> None: + with patch_variant_consequence_name(return_value=consequence_name): + variant_consequence = VariantConsequence({}) + assert variant_consequence.is_downstream() == expected + + @pytest.mark.parametrize( + "consequence_name,expected", + [ + ("foo", False), + ("upstream_gene_variant", True), + ], + ) + def test_is_upstream(self, consequence_name: str, expected: bool) -> None: + with patch_variant_consequence_name(return_value=consequence_name): + variant_consequence = VariantConsequence({}) + assert variant_consequence.is_upstream() == expected + + @pytest.mark.parametrize( + "consequence_name,expected", + [ + ("foo", False), + ("3_prime_UTR_variant", True), + ], + ) + def test_is_three_prime_utr(self, consequence_name: str, expected: bool) -> None: + with patch_variant_consequence_name(return_value=consequence_name): + variant_consequence = VariantConsequence({}) + assert variant_consequence.is_three_prime_utr() == expected + + @pytest.mark.parametrize( + "consequence_name,expected", + [ + ("foo", False), + ("5_prime_UTR_variant", True), + ], + ) + def test_is_five_prime_utr(self, consequence_name: str, expected: bool) -> None: + with patch_variant_consequence_name(return_value=consequence_name): + variant_consequence = VariantConsequence({}) + assert variant_consequence.is_five_prime_utr() == expected + + +class TestTranscript: + + SOME_EXON = "4/5" + SOME_INTRON = "4/4" + SOME_DISTANCE = "1234" + SOME_FEATURE = "amazing" + SOME_HIGH_IMPACT_CONSEQUENCE_NAME = "a_big_one" + SOME_LOW_IMPACT_CONSEQUENCE_NAME = "no_big_deal" + SOME_HIGH_IMPACT_CONSEQUENCE = { + "impact": "HIGH", + "var_conseq_name": SOME_HIGH_IMPACT_CONSEQUENCE_NAME, + } + SOME_LOW_IMPACT_CONSEQUENCE = { + "impact": "LOW", + "var_conseq_name": SOME_LOW_IMPACT_CONSEQUENCE_NAME, + } + SOME_TRANSCRIPT_PROPERTIES = { + "csq_exon": SOME_EXON, + "csq_feature": SOME_FEATURE, + "csq_consequence": [SOME_HIGH_IMPACT_CONSEQUENCE, SOME_LOW_IMPACT_CONSEQUENCE], + } + SOME_CANONICAL_TRANSCRIPT_PROPERTIES = {"csq_canonical": True} + SOME_MOST_SEVERE_TRANSCRIPT_PROPERTIES = {"csq_most_severe": True} + SOME_INTRON_TRANSCRIPT_PROPERTIES = {"csq_intron": SOME_INTRON} + SOME_DISTANCE_TRANSCRIPT_PROPERTIES = {"csq_distance": SOME_DISTANCE} + + def get_transcript(self, properties: JsonObject) -> Transcript: + return Transcript(properties) + + @pytest.mark.parametrize( + "transcript_properties,expected", + [ + (SOME_TRANSCRIPT_PROPERTIES, False), + (SOME_CANONICAL_TRANSCRIPT_PROPERTIES, True), + ], + ) + def test_is_canonical( + self, transcript_properties: JsonObject, expected: bool + ) -> None: + transcript = self.get_transcript(transcript_properties) + assert transcript.is_canonical() == expected + + @pytest.mark.parametrize( + "transcript_properties,expected", + [ + (SOME_TRANSCRIPT_PROPERTIES, False), + (SOME_MOST_SEVERE_TRANSCRIPT_PROPERTIES, True), + ], + ) + def test_is_most_severe( + self, transcript_properties: JsonObject, expected: bool + ) -> None: + transcript = self.get_transcript(transcript_properties) + assert transcript.is_most_severe() == expected + + @pytest.mark.parametrize( + "transcript_properties,expected", + [ + (SOME_TRANSCRIPT_PROPERTIES, SOME_FEATURE), + (SOME_MOST_SEVERE_TRANSCRIPT_PROPERTIES, ""), + ], + ) + def test_get_feature( + self, transcript_properties: JsonObject, expected: str + ) -> None: + transcript = self.get_transcript(transcript_properties) + assert transcript.get_feature() == expected + + @pytest.mark.parametrize("most_severe_consequence", [None, "some_consequence"]) + def test_get_location(self, most_severe_consequence: Any) -> None: + with patch_transcript_get_most_severe_consequence( + return_value=most_severe_consequence + ): + with patch_transcript_get_location_by_most_severe_consequence() as mock_get_location_by_consequence: + transcript = self.get_transcript({}) + result = transcript.get_location() + if most_severe_consequence: + mock_get_location_by_consequence.assert_called_once_with( + most_severe_consequence + ) + assert result == mock_get_location_by_consequence.return_value + else: + mock_get_location_by_consequence.assert_not_called() + assert result == "" + + @pytest.mark.parametrize( + "transcript_properties,expected", + [ + (SOME_MOST_SEVERE_TRANSCRIPT_PROPERTIES, None), + (SOME_TRANSCRIPT_PROPERTIES, SOME_HIGH_IMPACT_CONSEQUENCE), + ], + ) + def test_get_most_severe_consequence( + self, transcript_properties: JsonObject, expected: Union[JsonObject, None] + ) -> None: + transcript = self.get_transcript(transcript_properties) + consequence = transcript._get_most_severe_consequence() + if not expected: + assert consequence == expected + else: + assert consequence.get_properties() == expected + + @pytest.mark.parametrize( + ( + "transcript_properties,consequence_upstream,consequence_downstream," + "consequence_3_utr,consequence_5_utr,expected" + ), + [ + ({}, False, False, False, False, ""), + ( + SOME_TRANSCRIPT_PROPERTIES, + False, + False, + False, + False, + f"Exon {SOME_EXON}", + ), + ( + SOME_TRANSCRIPT_PROPERTIES, + False, + False, + True, + False, + f"Exon {SOME_EXON} (3' UTR)", + ), + ( + SOME_TRANSCRIPT_PROPERTIES, + False, + False, + False, + True, + f"Exon {SOME_EXON} (5' UTR)", + ), + ( + SOME_INTRON_TRANSCRIPT_PROPERTIES, + False, + False, + False, + False, + f"Intron {SOME_INTRON}", + ), + ( + SOME_INTRON_TRANSCRIPT_PROPERTIES, + False, + False, + True, + False, + f"Intron {SOME_INTRON} (3' UTR)", + ), + ( + SOME_INTRON_TRANSCRIPT_PROPERTIES, + False, + False, + False, + True, + f"Intron {SOME_INTRON} (5' UTR)", + ), + (SOME_DISTANCE_TRANSCRIPT_PROPERTIES, False, False, False, False, ""), + ( + SOME_DISTANCE_TRANSCRIPT_PROPERTIES, + True, + False, + False, + False, + f"{SOME_DISTANCE} bp upstream", + ), + ( + SOME_DISTANCE_TRANSCRIPT_PROPERTIES, + False, + True, + False, + False, + f"{SOME_DISTANCE} bp downstream", + ), + (SOME_DISTANCE_TRANSCRIPT_PROPERTIES, False, False, False, True, ""), + ], + ) + def test_get_location_by_most_severe_consequence( + self, + transcript_properties: JsonObject, + consequence_upstream: bool, + consequence_downstream: bool, + consequence_3_utr: bool, + consequence_5_utr: bool, + expected: str, + ) -> None: + transcript = self.get_transcript(transcript_properties) + variant_consequence = mock_variant_consequence( + is_upstream=consequence_upstream, + is_downstream=consequence_downstream, + is_three_prime_utr=consequence_3_utr, + is_five_prime_utr=consequence_5_utr, + ) + result = transcript._get_location_by_most_severe_consequence( + variant_consequence + ) + assert result == expected + + @pytest.mark.parametrize( + "transcript_properties,expected", + [ + ({}, ""), + ( + SOME_TRANSCRIPT_PROPERTIES, + f"{SOME_HIGH_IMPACT_CONSEQUENCE_NAME}, {SOME_LOW_IMPACT_CONSEQUENCE_NAME}", + ), + ], + ) + def test_get_consequence_names( + self, transcript_properties: JsonObject, expected: str + ) -> None: + transcript = self.get_transcript(transcript_properties) + result = transcript.get_consequence_names() + assert result == expected + + +class TestVariant: + + SOME_CANONICAL_TRANSCRIPT = mock_transcript(canonical=True) + SOME_MOST_SEVERE_TRANSCRIPT = mock_transcript(most_severe=True) + SOME_TRANSCRIPTS = [SOME_CANONICAL_TRANSCRIPT, SOME_MOST_SEVERE_TRANSCRIPT] + SOME_GNOMAD_V3_POPULATION = "csq_gnomadg_af-ami" + SOME_GNOMAD_V2_POPULATION = "csq_gnomade2_af-amr" + SOME_VARIANT_PROPERTIES = { + "csq_gnomadg_af_popmax": 0.123, + "csq_gnomadg_af-ami": 0.123, + "csq_gnomadg_af-afr": 0.111, + "csq_gnomade2_af_popmax": 0.987, + "csq_gnomade2_af-amr": 0.987, + "csq_gnomade2_af-asj": 0.9, + } + + def get_variant(self, properties: Optional[JsonObject] = None) -> Variant: + if properties is None: + properties = self.SOME_VARIANT_PROPERTIES + return Variant(properties) + + def test_get_canonical_transcript(self) -> None: + with patch_variant_get_transcripts(return_value=self.SOME_TRANSCRIPTS): + variant = self.get_variant() + assert variant._get_canonical_transcript() == self.SOME_CANONICAL_TRANSCRIPT + + def test_get_most_severe_transcript(self) -> None: + with patch_variant_get_transcripts(return_value=self.SOME_TRANSCRIPTS): + variant = self.get_variant() + assert ( + variant._get_most_severe_transcript() + == self.SOME_MOST_SEVERE_TRANSCRIPT + ) + + @pytest.mark.parametrize("canonical_transcript", [None, mock_transcript()]) + def test_get_canonical_transcript_feature( + self, canonical_transcript: Union[mock.MagicMock, None] + ) -> None: + with patch_variant_get_canonical_transcript(return_value=canonical_transcript): + variant = self.get_variant() + if canonical_transcript: + expected = canonical_transcript.get_feature.return_value + else: + expected = "" + assert variant.get_canonical_transcript_feature() == expected + + @pytest.mark.parametrize("most_severe_transcript", [None, mock_transcript()]) + def test_get_most_severe_transcript_feature( + self, most_severe_transcript: Union[mock.MagicMock, None] + ) -> None: + with patch_variant_get_most_severe_transcript( + return_value=most_severe_transcript + ): + variant = self.get_variant() + if most_severe_transcript: + expected = most_severe_transcript.get_feature.return_value + else: + expected = "" + assert variant.get_most_severe_transcript_feature() == expected + + @pytest.mark.parametrize("canonical_transcript", [None, mock_transcript()]) + def test_get_canonical_transcript_consequence_names( + self, canonical_transcript: Union[mock.MagicMock, None] + ) -> None: + with patch_variant_get_canonical_transcript(return_value=canonical_transcript): + variant = self.get_variant() + if canonical_transcript: + expected = canonical_transcript.get_consequence_names.return_value + else: + expected = "" + assert variant.get_canonical_transcript_consequence_names() == expected + + @pytest.mark.parametrize("most_severe_transcript", [None, mock_transcript()]) + def test_get_most_severe_transcript_consequence_names( + self, most_severe_transcript: Union[mock.MagicMock, None] + ) -> None: + with patch_variant_get_most_severe_transcript( + return_value=most_severe_transcript + ): + variant = self.get_variant() + if most_severe_transcript: + expected = most_severe_transcript.get_consequence_names.return_value + else: + expected = "" + assert variant.get_most_severe_transcript_consequence_names() == expected + + @pytest.mark.parametrize("canonical_transcript", [None, mock_transcript()]) + def test_get_canonical_transcript_location( + self, canonical_transcript: Union[mock.MagicMock, None] + ) -> None: + with patch_variant_get_canonical_transcript(return_value=canonical_transcript): + variant = self.get_variant() + if canonical_transcript: + expected = canonical_transcript.get_location.return_value + else: + expected = "" + assert variant.get_canonical_transcript_location() == expected + + @pytest.mark.parametrize("most_severe_transcript", [None, mock_transcript()]) + def test_get_most_severe_transcript_location( + self, most_severe_transcript: Union[mock.MagicMock, None] + ) -> None: + with patch_variant_get_most_severe_transcript( + return_value=most_severe_transcript + ): + variant = self.get_variant() + if most_severe_transcript: + expected = most_severe_transcript.get_location.return_value + else: + expected = "" + assert variant.get_most_severe_transcript_location() == expected + + @pytest.mark.parametrize( + "variant_properties,expected", [({}, ""), (SOME_VARIANT_PROPERTIES, "Amish")] + ) + def test_get_gnomad_v3_popmax_population( + self, variant_properties: JsonObject, expected: str + ) -> None: + variant = self.get_variant(properties=variant_properties) + result = variant.get_gnomad_v3_popmax_population() + assert result == expected + + @pytest.mark.parametrize( + "variant_properties,expected", [({}, ""), (SOME_VARIANT_PROPERTIES, "Latino")] + ) + def test_get_gnomad_v2_popmax_population( + self, variant_properties: JsonObject, expected: str + ) -> None: + variant = self.get_variant(properties=variant_properties) + result = variant.get_gnomad_v2_popmax_population() + assert result == expected + + +class TestVariantSample: + + SOME_VARIANT_PROPERTIES = {"foo": "bar"} + SOME_PROJECT = {"name": "something"} + SOME_NOTE_OF_SAME_PROJECT_PROPERTIES = {"project": SOME_PROJECT} + ANOTHER_NOTE_OF_SAME_PROJECT_PROPERTIES = {"project": SOME_PROJECT, "fu": "bur"} + SOME_NOTE_OF_DIFFERENT_PROJECT_PROPERTIES = {"project": "something"} + SOME_PROPERTY_FOR_NOTE = "a_note_field" + SOME_VARIANT_SAMPLE_PROPERTIES = { + "variant": SOME_VARIANT_PROPERTIES, + "project": SOME_PROJECT, + } + + def get_variant_sample( + self, properties: Optional[JsonObject] = None + ) -> VariantSample: + if properties is None: + properties = self.SOME_VARIANT_SAMPLE_PROPERTIES + return VariantSample(properties) + + @pytest.mark.parametrize( + "variant,expected_variant", + [("", False), ("something", False), (SOME_VARIANT_PROPERTIES, True)], + ) + def test_get_variant( + self, variant: Union[JsonObject, None], expected_variant: bool + ) -> None: + with patch_variant_sample_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample._get_variant() + if expected_variant: + assert isinstance(result, Variant) + assert result.get_properties() == variant + else: + assert result is None + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_canonical_transcript_feature( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_canonical_transcript_feature() + if variant: + assert result == variant.get_canonical_transcript_feature.return_value + else: + assert result == "" + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_canonical_transcript_location( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_canonical_transcript_location() + if variant: + assert result == variant.get_canonical_transcript_location.return_value + else: + assert result == "" + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_canonical_transcript_consequence_names( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_canonical_transcript_consequence_names() + if variant: + assert ( + result + == variant.get_canonical_transcript_consequence_names.return_value + ) + else: + assert result == "" + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_most_severe_transcript_feature( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_most_severe_transcript_feature() + if variant: + assert result == variant.get_most_severe_transcript_feature.return_value + else: + assert result == "" + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_most_severe_transcript_location( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_most_severe_transcript_location() + if variant: + assert ( + result == variant.get_most_severe_transcript_location.return_value + ) + else: + assert result == "" + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_most_severe_transcript_consequence_names( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_most_severe_transcript_consequence_names() + if variant: + assert ( + result + == variant.get_most_severe_transcript_consequence_names.return_value + ) + else: + assert result == "" + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_gnomad_v3_popmax_population( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_gnomad_v3_popmax_population() + if variant: + assert result == variant.get_gnomad_v3_popmax_population.return_value + else: + assert result == "" + + @pytest.mark.parametrize("variant", [None, mock_variant()]) + def test_get_gnomad_v2_popmax_population( + self, variant: Union[Variant, None] + ) -> None: + with patch_variant_sample_get_variant(return_value=variant): + variant_sample = self.get_variant_sample() + result = variant_sample.get_gnomad_v2_popmax_population() + if variant: + assert result == variant.get_gnomad_v2_popmax_population.return_value + else: + assert result == "" + + @pytest.mark.parametrize( + "notes_at_property,expected_note", + [ + ([], None), + ([SOME_NOTE_OF_DIFFERENT_PROJECT_PROPERTIES], None), + ( + [ + SOME_NOTE_OF_SAME_PROJECT_PROPERTIES, + SOME_NOTE_OF_DIFFERENT_PROJECT_PROPERTIES, + ], + SOME_NOTE_OF_SAME_PROJECT_PROPERTIES, + ), + ( + [ + ANOTHER_NOTE_OF_SAME_PROJECT_PROPERTIES, + SOME_NOTE_OF_SAME_PROJECT_PROPERTIES, + ], + SOME_NOTE_OF_SAME_PROJECT_PROPERTIES, + ), + ], + ) + def test_get_most_recent_note_of_same_project_project( + self, + notes_at_property: List[JsonObject], + expected_note: Union[JsonObject, None], + ) -> None: + variant_sample_properties = { + **self.SOME_VARIANT_SAMPLE_PROPERTIES, + self.SOME_PROPERTY_FOR_NOTE: notes_at_property, + } + variant_sample = self.get_variant_sample(variant_sample_properties) + result = variant_sample.get_most_recent_note_of_same_project_from_property( + self.SOME_PROPERTY_FOR_NOTE + ) + if expected_note is not None: + assert isinstance(result, Note) + assert result.get_properties() == expected_note + else: + assert result is None + + +class TestVariantSampleList: + + SOME_UUID = "uuid_1" + ANOTHER_UUID = "uuid_2" + SOME_VARIANT_SAMPLE_SELECTION = {"variant_sample_item": SOME_UUID} + ANOTHER_VARIANT_SAMPLE_SELECTION = {"variant_sample_item": ANOTHER_UUID} + SOME_VARIANT_SAMPLE_LIST_PROPERTIES = { + "variant_samples": [ + SOME_VARIANT_SAMPLE_SELECTION, + ANOTHER_VARIANT_SAMPLE_SELECTION, + ] + } + + def get_variant_sample_list( + self, properties: Optional[JsonObject] = None + ) -> VariantSampleList: + if properties is None: + properties = self.SOME_VARIANT_SAMPLE_LIST_PROPERTIES + return VariantSampleList(properties) + + @pytest.mark.parametrize( + "variant_sample_list_properties,expected", + [({}, []), (SOME_VARIANT_SAMPLE_LIST_PROPERTIES, [SOME_UUID, ANOTHER_UUID])], + ) + def test_get_variant_samples( + self, variant_sample_list_properties: JsonObject, expected: List[str] + ): + variant_sample_list = self.get_variant_sample_list( + variant_sample_list_properties + ) + result = variant_sample_list.get_variant_samples() + assert result == expected diff --git a/src/encoded/tests/test_search_export_tsv.py b/src/encoded/tests/test_search_export_tsv.py deleted file mode 100644 index 76d51c2690..0000000000 --- a/src/encoded/tests/test_search_export_tsv.py +++ /dev/null @@ -1,118 +0,0 @@ -import json -import pytest -import csv - -pytestmark = [pytest.mark.working, pytest.mark.schema, pytest.mark.search, pytest.mark.workbook] - - -### IMPORTANT -# uses the inserts in ./data/workbook_inserts -# design your tests accordingly - - -def check_spreadsheet_rows(result_rows, colname_to_index, row_start=0): - - # Check presence of columns in the doc - assert result_rows[row_start][1] == "Chrom (hg38)" - assert result_rows[row_start][2] == "Pos (hg38)" - assert result_rows[row_start][16] == "Canonical transcript ID" - assert result_rows[row_start][17] == "Canonical transcript location" - assert result_rows[row_start][25] == "Variant Quality" - - # Add all test cases to here - expected_data = [ - { - "Chrom (hg38)": "1", - "Pos (hg38)": "2030666", - "Chrom (hg19)": "1", - "Pos (hg19)": "1962105", - "Canonical transcript ID": "ENST00000378585", - "Canonical transcript location": "Exon 9/9 (3′ UTR)", - "Variant Quality": "688.12", - "Genotype Quality": "99", - "gnomADv3 popmax population": "East Asian", - "LOEUF": "0.245" - }, - { - "Chrom (hg38)": "16", - "Pos (hg38)": "23630011", - "Canonical transcript ID": "ENST00000261584", - "Canonical transcript location": "Exon 5/13", - "Variant Quality": "261.94", - "Genotype Quality": "24", - "gnomADv3 popmax population": "", - "LOEUF": "1.006" - } - ] - - for data_idx, ed in enumerate(expected_data): - for name, value in ed.items(): - # Data starts 2 rows after (header + description) - assert result_rows[data_idx + row_start + 2][colname_to_index[name]] == value - - -def test_filtering_tab(workbook, html_es_testapp): - - compound_filterset_request_body = { - "search_type":"VariantSample", - # In production it's probably more likely we sort by -date_created (descending), however sort=date_created - # simplifies keeping order of workbook-inserts in sync. - "global_flags":"CALL_INFO=SAM10254-S1&file=GAPFI3EBH4X2&additional_facet=proband_only_inheritance_modes&sort=date_created", - "intersect": False, - "filter_blocks":[ - { - "query":"associated_genotype_labels.proband_genotype_label=Heterozygous&associated_genelists=Breast+Cancer+%2828%29&variant.genes.genes_most_severe_consequence.impact=MODERATE&variant.genes.genes_most_severe_consequence.impact=HIGH", - "flags_applied":[ - - ] - }, - { - "query":"GQ.from=60&GQ.to=99&associated_genotype_labels.proband_genotype_label=Heterozygous&associated_genelists=Familial+Cancer+%28148%29&variant.csq_clinvar_clnsig=Uncertain_significance&variant.csq_clinvar_clnsig=Pathogenic&variant.csq_gnomadg_af.from=0&variant.csq_gnomadg_af.to=0.001&variant.genes.genes_most_severe_consequence.impact=MODERATE&variant.genes.genes_most_severe_consequence.impact=HIGH", - "flags_applied":[ - - ] - }, - { - "query":"variant.csq_gnomade2_af.from=0&variant.csq_gnomade2_af.to=0.001&variant.csq_gnomadg_af.from=0&variant.csq_gnomadg_af.to=0.001", - "flags_applied":[ - - ] - } - ] - } - - res = html_es_testapp.post( - '/variant-sample-search-spreadsheet/', - { - "compound_search_request": json.dumps(compound_filterset_request_body) - }, - content_type="application/x-www-form-urlencoded" - ) - - assert 'text/tsv' in res.content_type - - # All values are of type string when parsed below. - result_rows = list(csv.reader(res.body.decode('utf-8').split('\n'), delimiter='\t')) - colname_to_index = { col_name: col_idx for col_idx, col_name in enumerate(result_rows[6]) } - - check_spreadsheet_rows(result_rows, colname_to_index, row_start=6) - - - - -def test_interpretation_tab(workbook, html_es_testapp): - - res = html_es_testapp.get( - '/variant-sample-lists/292250e7-5cb7-4543-85b2-80cd318287b2/@@spreadsheet/?file_format=csv', - ) - - # Should infer from suggested filename ending in .tsv - assert 'text/csv' in res.content_type - - # All values are of type string when parsed below. - result_rows = list(csv.reader(res.body.decode('utf-8').split('\n'), delimiter=",")) - - colname_to_index = { col_name: col_idx for col_idx, col_name in enumerate(result_rows[0]) } - - check_spreadsheet_rows(result_rows, colname_to_index) - diff --git a/src/encoded/tests/test_types_variant_sample_list.py b/src/encoded/tests/test_types_variant_sample_list.py new file mode 100644 index 0000000000..4bb0dd7a63 --- /dev/null +++ b/src/encoded/tests/test_types_variant_sample_list.py @@ -0,0 +1,24 @@ +import pytest +from webtest.app import TestApp + +from .test_batch_download import EXPECTED_VARIANT_SAMPLE_SPREADSHEET_COLUMNS, parse_spreadsheet_response + + +@pytest.mark.workbook +def test_variant_sample_list_spreadsheet(es_testapp: TestApp, workbook: None) -> None: + """Integrated test of associated VariantSample spreadsheet download. + + Ensure all fields present on at least one VariantSample included in + the spreadsheet. + + For testing simplicity, keeping VariantSamples included in this + spreadsheet identical with those in the VariantSample from search + spreadsheet integrated test. + """ + item_uuid = "292250e7-5cb7-4543-85b2-80cd318287b2" + item_spreadsheet_endpoint = f"/variant-sample-lists/{item_uuid}/@@spreadsheet/?file_format=tsv" + response = es_testapp.get(item_spreadsheet_endpoint, status=200) + spreadsheet_rows = parse_spreadsheet_response(response) + spreadsheet_columns = list(zip(*spreadsheet_rows)) + assert len(spreadsheet_rows) == 4 + assert spreadsheet_columns == EXPECTED_VARIANT_SAMPLE_SPREADSHEET_COLUMNS diff --git a/src/encoded/tests/utils.py b/src/encoded/tests/utils.py index b502aef893..22ad88f8c0 100644 --- a/src/encoded/tests/utils.py +++ b/src/encoded/tests/utils.py @@ -1,5 +1,7 @@ import re -from typing import Any, Dict, Optional +from contextlib import contextmanager +from typing import Any, Dict, Iterator, Optional +from unittest import mock from webtest.app import TestApp @@ -27,6 +29,27 @@ def make_atid(uuid, item_type="sample-processing"): return f"/{pluralize(item_type)}/{uuid}/" +AN_UNLIKELY_RETURN_VALUE = "unlikely return value" + + +@contextmanager +def patch_context( + to_patch: object, + return_value: Any = AN_UNLIKELY_RETURN_VALUE, + **kwargs, +) -> Iterator[mock.MagicMock]: + if isinstance(to_patch, property): + to_patch = to_patch.fget + new_callable = mock.PropertyMock + else: + new_callable = mock.MagicMock + target = f"{to_patch.__module__}.{to_patch.__qualname__}" + with mock.patch(target, new_callable=new_callable, **kwargs) as mocked_item: + if return_value != AN_UNLIKELY_RETURN_VALUE: + mocked_item.return_value = return_value + yield mocked_item + + def get_identifier( testapp: TestApp, identifier: str, frame: str = "object" ) -> Dict[str, Any]: diff --git a/src/encoded/types/variant.py b/src/encoded/types/variant.py index 91d2abc888..b420f6d4ff 100644 --- a/src/encoded/types/variant.py +++ b/src/encoded/types/variant.py @@ -1,34 +1,40 @@ -# import boto3 -# import csv import datetime import io import json -import negspy.coordinates as nc import os +from typing import Iterator, List +from urllib.parse import parse_qs, urlparse + +import negspy.coordinates as nc import pytz import structlog - from dcicutils.misc_utils import ignorable, ignored -from math import inf from pyramid.httpexceptions import HTTPBadRequest, HTTPNotModified, HTTPServerError, HTTPTemporaryRedirect -# from pyramid.request import Request -from pyramid.response import Response +from pyramid.request import Request from pyramid.settings import asbool from pyramid.traversal import find_resource from pyramid.view import view_config from snovault import calculated_property, collection, load_schema # , TYPES from snovault.calculated import calculate_properties from snovault.embed import make_subrequest -from snovault.util import simple_path_ids, debug_log, IndexSettings -from urllib.parse import parse_qs, urlparse - -from ..batch_download_utils import stream_tsv_output, convert_item_to_sheet_dict +from snovault.util import debug_log, IndexSettings + +from .. import custom_embed +from ..batch_download import ( + get_spreadsheet_response, + get_timestamp, + get_variant_sample_rows, + validate_spreadsheet_file_format, + VariantSampleSpreadsheet, +) +from ..batch_download_utils import SpreadsheetRequest from ..custom_embed import CustomEmbed +from ..item_models import VariantSampleList as VariantSampleListModel from ..ingestion.common import CGAP_CORE_PROJECT from ..inheritance_mode import InheritanceMode from ..search.search import get_iterable_search_results from ..types.base import Item, get_item_or_none -from ..util import resolve_file_path, build_s3_presigned_get_url, convert_integer_to_comma_string +from ..util import JsonObject, resolve_file_path, build_s3_presigned_get_url, convert_integer_to_comma_string log = structlog.getLogger(__name__) @@ -188,6 +194,7 @@ def build_variant_sample_embedded_list(): """ embedded_list = SHARED_VARIANT_SAMPLE_EMBEDS + [ "cmphet.*", + "variant.genes.genes_most_severe_gene.gene_notes.@id", ] with io.open(resolve_file_path('schemas/variant_embeds.json'), 'r') as fd: extend_embedded_list(embedded_list, fd, 'variant', prefix='variant.') @@ -1343,365 +1350,98 @@ def order_delete_selections(context, request): } -@view_config(name='spreadsheet', context=VariantSampleList, request_method='GET', - permission='view') -@debug_log -def variant_sample_list_spreadsheet(context, request): - - file_format = request.GET.get("file_format", None) - case_accession = request.GET.get("case_accession", context.properties.get("created_for_case")) - - if not file_format: - file_format = "tsv" - elif file_format not in { "tsv", "csv" }: # TODO: Add support for xslx. - raise HTTPBadRequest("Expected a valid `file_format` such as TSV or CSV.") - - - timestamp = datetime.datetime.now(pytz.utc).isoformat()[:-13] + "Z" - suggested_filename = (case_accession or "case") + "-interpretation-" + timestamp + "." + file_format - - - variant_sample_uuids = [ vso["variant_sample_item"] for vso in context.properties.get("variant_samples", []) ] - spreadsheet_mappings = get_spreadsheet_mappings(request) - fields_to_embed = get_fields_to_embed(spreadsheet_mappings) - - - def load_variant_sample(vs_uuid): - ''' - We want to grab datastore=database version of Items here since is likely that user has _just_ finished making - an edit when they decide to export the spreadsheet from the InterpretationTab UI. - ''' - vs_embedding_instance = CustomEmbed(request, vs_uuid, embed_props={ "requested_fields": fields_to_embed }) - result = vs_embedding_instance.result - return result - - def vs_dicts_generator(): - for vs_uuid in variant_sample_uuids: - vs_result = load_variant_sample(vs_uuid) - yield convert_item_to_sheet_dict(vs_result, spreadsheet_mappings) - - - return Response( - app_iter = stream_tsv_output( - vs_dicts_generator(), - spreadsheet_mappings, - file_format - ), - headers={ - 'X-Accel-Buffering': 'no', - # 'Content-Encoding': 'utf-8', # Disabled so that Python unit test may work (TODO: Look into more?) - 'Content-Disposition': 'attachment; filename=' + suggested_filename, - 'Content-Type': 'text/' + file_format, - 'Content-Description': 'File Transfer', - 'Cache-Control': 'no-store' - } - ) - - -############################################################ -### Spreadsheet Generation for Variant Sample Item Lists ### -############################################################ - - -POPULATION_SUFFIX_TITLE_TUPLES = [ - ("afr", "African-American/African"), - ("ami", "Amish"), - ("amr", "Latino"), - ("asj", "Ashkenazi Jewish"), - ("eas", "East Asian"), - ("fin", "Finnish"), - ("mid", "Middle Eastern"), - ("nfe", "Non-Finnish European"), - ("oth", "Other Ancestry"), - ("sas", "South Asian") +VARIANT_SAMPLE_FIELDS_TO_EMBED_FOR_SPREADSHEET = [ + "*", + "variant.*", + "variant.interpretations.*", + "variant.discovery_interpretations.*", + "variant.variant_notes.*", + "variant.genes.genes_most_severe_gene.gene_notes.*", + "variant.transcript.*", + "variant.transcript.csq_consequence.*", ] -def get_spreadsheet_mappings(request = None): - - def get_boolean_transcript_field(variant_sample, field): - variant = variant_sample.get("variant", {}) - for transcript in variant.get("transcript", []): - if transcript.get(field, False) is True: - return transcript - return None - def get_canonical_transcript(variant_sample): - return get_boolean_transcript_field(variant_sample, "csq_canonical") +@view_config( + name='spreadsheet', + context=VariantSampleList, + request_method='GET', + permission='view', + validators=[validate_spreadsheet_file_format], +) +@debug_log +def variant_sample_list_spreadsheet(context: VariantSampleList, request: Request): + """Download spreadsheet for VariantSamples in VariantSampleList.""" + spreadsheet_request = SpreadsheetRequest(request) + file_format = spreadsheet_request.get_file_format() + file_name = get_variant_sample_spreadsheet_file_name(context, spreadsheet_request) + items_for_spreadsheet = get_embedded_items(context, request) + spreadsheet_rows = get_variant_sample_rows( + items_for_spreadsheet, spreadsheet_request=spreadsheet_request, embed_additional_items=False + ) + return get_spreadsheet_response(file_name, spreadsheet_rows, file_format) - def get_most_severe_transcript(variant_sample): - return get_boolean_transcript_field(variant_sample, "csq_most_severe") - def get_most_severe_consequence(variant_sample_transcript): - ''' Used only for "Location" ''' - csq_consequences = variant_sample_transcript.get("csq_consequence", []) - if not csq_consequences: - return None - impact_map = { - "HIGH" : 0, - "MODERATE" : 1, - "LOW" : 2, - "MODIFIER" : 3 - } - most_severe_impact_val = inf - most_severe_consequence = None - for consequence in csq_consequences: - impact_val = impact_map[consequence["impact"]] - if impact_val < most_severe_impact_val: - most_severe_consequence = consequence - most_severe_impact_val = impact_val - - return most_severe_consequence - - - def canonical_transcript_csq_feature(variant_sample): - ''' Returns `variant.transcript.csq_feature` ''' - canonical_transcript = get_canonical_transcript(variant_sample) - if canonical_transcript: - return canonical_transcript.get("csq_feature", None) - return None +def get_variant_sample_spreadsheet_file_name( + context: VariantSampleList, spreadsheet_request: SpreadsheetRequest +) -> str: + file_format = spreadsheet_request.get_file_format() + case_title = get_case_title(context, spreadsheet_request) + timestamp = get_timestamp() + return f"{case_title}-interpretation-{timestamp}.{file_format}" - def most_severe_transcript_csq_feature(variant_sample): - ''' Returns `variant.transcript.csq_feature` ''' - most_severe_transcript = get_most_severe_transcript(variant_sample) - if most_severe_transcript: - return most_severe_transcript.get("csq_feature", None) - return None - # TODO: Consider making `canonical_transcript_location` + `most_severe_transcript_location` as calculated properties - def location_name(transcript): - most_severe_consequence = get_most_severe_consequence(transcript) - consequence_name = most_severe_consequence["var_conseq_name"].lower() if most_severe_consequence is not None and "var_conseq_name" in most_severe_consequence else None - - return_str = None - - csq_exon = transcript.get("csq_exon", None) - csq_intron = transcript.get("csq_intron", None) - csq_distance = transcript.get("csq_distance", None) - - if csq_exon is not None: - return_str = "Exon " + csq_exon - elif csq_intron is not None: - return_str = "Intron " + csq_intron - elif csq_distance is not None and consequence_name is not None: - if consequence_name == "downstream_gene_variant": - return_str = csq_distance + "bp downstream" - elif consequence_name == "upstream_gene_variant": - return_str = csq_distance + "bp upstream" - - if consequence_name == "3_prime_utr_variant": - if return_str: - return_str += " (3′ UTR)" - else: - return_str = "3′ UTR" - elif consequence_name == "5_prime_utr_variant": - if return_str: - return_str += " (5′ UTR)" - else: - return_str = "5′ UTR" +def get_case_title(context: VariantSampleList, spreadsheet_request: SpreadsheetRequest) -> str: + return ( + spreadsheet_request.get_case_accession() + or get_associated_case(context) + or "case" + ) - return return_str - def canonical_transcript_location(variant_sample): - canonical_transcript = get_canonical_transcript(variant_sample) - if not canonical_transcript: - return None - return location_name(canonical_transcript) +def get_associated_case(context: VariantSampleList) -> str: + properties = get_item_properties(context) + return VariantSampleListModel(properties).get_associated_case_accession() - def most_severe_transcript_location(variant_sample): - most_severe_transcript = get_most_severe_transcript(variant_sample) - if not most_severe_transcript: - return None - return location_name(most_severe_transcript) - def canonical_transcript_consequence_display_title(variant_sample): - canonical_transcript = get_canonical_transcript(variant_sample) - if not canonical_transcript: - return None - csq_consequences = canonical_transcript.get("csq_consequence", []) - if not csq_consequences: - return None - return ", ".join([ c["display_title"] for c in csq_consequences ]) +def get_embedded_items( + context: VariantSampleList, request: Request +) -> Iterator[JsonObject]: + variant_sample_uuids = get_variant_sample_uuids(context) + return (get_embedded_variant_sample(uuid, request) for uuid in variant_sample_uuids) - def most_severe_transcript_consequence_display_title(variant_sample): - most_severe_transcript = get_most_severe_transcript(variant_sample) - if not most_severe_transcript: - return None - csq_consequences = most_severe_transcript.get("csq_consequence", []) - if not csq_consequences: - return None - return ", ".join([ c["display_title"] for c in csq_consequences ]) - def gnomadv3_popmax_population(variant_sample): - variant = variant_sample.get("variant", {}) - csq_gnomadg_af_popmax = variant.get("csq_gnomadg_af_popmax") - if not csq_gnomadg_af_popmax: # Return None for 0, also. - return None - for pop_suffix, pop_name in POPULATION_SUFFIX_TITLE_TUPLES: - pop_val = variant.get("csq_gnomadg_af-" + pop_suffix) - if pop_val is not None and pop_val == csq_gnomadg_af_popmax: - return pop_name - return None +def get_item_properties(context: Item) -> JsonObject: + return context.properties - def gnomadv2_popmax_population(variant_sample): - variant = variant_sample.get("variant", {}) - csq_gnomade2_af_popmax = variant.get("csq_gnomade2_af_popmax") - if not csq_gnomade2_af_popmax: # Return None for 0, also. - return None - for pop_suffix, pop_name in POPULATION_SUFFIX_TITLE_TUPLES: - pop_val = variant.get("csq_gnomade2_af-" + pop_suffix) - if pop_val is not None and pop_val == csq_gnomade2_af_popmax: - return pop_name - return None - def get_most_recent_note_of_project(notes_iterable, project_at_id): - for note in reversed(list(notes_iterable)): - note_project_id = note["project"] - if isinstance(note_project_id, dict): - # We might get string OR @@embedded representation, e.g. if from search response. - note_project_id = note_project_id.get("@id") - if project_at_id == note["project"]: - return note - return None +def get_variant_sample_uuids(context: VariantSampleList) -> List[str]: + variant_sample_list_properties = get_item_properties(context) + variant_sample_list = VariantSampleListModel(variant_sample_list_properties) + return variant_sample_list.get_variant_samples() - def own_project_note_factory(note_field_of_vs, note_field): - def callable(variant_sample): - notes_iterable = simple_path_ids(variant_sample, note_field_of_vs) - vs_project_at_id = variant_sample.get("project") - if isinstance(vs_project_at_id, dict): - # We might get string OR @@embedded representation, e.g. if from search response. - vs_project_at_id = vs_project_at_id.get("@id") - if not vs_project_at_id: - return None +def get_embedded_variant_sample(variant_sample_identifier: str, request: Request) -> JsonObject: + embedding_parameters = get_embedding_parameters() + return custom_embed.CustomEmbed( + request, variant_sample_identifier, embedding_parameters + ).get_embedded_item() - note_item = get_most_recent_note_of_project(notes_iterable, vs_project_at_id) - if note_item: - return note_item.get(note_field) - else: - return None +def get_embedding_parameters() -> JsonObject: + fields_to_embed = get_fields_to_embed() + return {custom_embed.REQUESTED_FIELDS: fields_to_embed} - return callable +def get_fields_to_embed() -> List[str]: + fields_from_spreadsheet = get_fields_to_embed_from_spreadsheet() + return fields_from_spreadsheet + VARIANT_SAMPLE_FIELDS_TO_EMBED_FOR_SPREADSHEET - # portal_root_url = request.resource_url(request.root)[:-1] +def get_fields_to_embed_from_spreadsheet() -> List[str]: + spreadsheet_columns = VariantSampleSpreadsheet.get_spreadsheet_columns() return [ - ## Column Title | CGAP Field (if not custom function) | Description - ## --------------------------------------- | ----------------------------------------------------------- | -------------------------------------------------------------------------- - ("ID", "@id", "URL path to the Sample Variant on this row"), - ("Chrom (hg38)", "variant.CHROM", "Chromosome (hg38 assembly)"), - ("Pos (hg38)", "variant.POS", "Start Position (hg38 assembly)"), - ("Chrom (hg19)", "variant.hg19_chr", "Chromosome (hg19 assembly)"), - ("Pos (hg19)", "variant.hg19_pos", "Start Position (hg19 assembly)"), - ("Ref", "variant.REF", "Reference Nucleotide"), - ("Alt", "variant.ALT", "Alternate Nucleotide"), - ("Proband genotype", "associated_genotype_labels.proband_genotype_label", "Proband Genotype"), - ("Mother genotype", "associated_genotype_labels.mother_genotype_label", "Mother Genotype"), - ("Father genotype", "associated_genotype_labels.father_genotype_label", "Father Genotype"), - ("HGVSG", "variant.hgvsg", "HGVS genomic nomenclature"), - ("HGVSC", "variant.genes.genes_most_severe_hgvsc", "HGVS cPos nomenclature"), - ("HGVSP", "variant.genes.genes_most_severe_hgvsp", "HGVS pPos nomenclature"), - ("dbSNP ID", "variant.ID", "dbSNP ID of variant"), - ("Genes", "variant.genes.genes_most_severe_gene.display_title", "Gene symbol(s)"), - ("Gene type", "variant.genes.genes_most_severe_gene.gene_biotype", "Type of Gene"), - # ONLY FOR variant.transcript.csq_canonical=true - ("Canonical transcript ID", canonical_transcript_csq_feature, "Ensembl ID of canonical transcript of gene variant is in"), - # ONLY FOR variant.transcript.csq_canonical=true; use `variant.transcript.csq_intron` if `variant.transcript.csq_exon` not present (display as in annotation space: eg. exon 34/45 or intron 4/7) - ("Canonical transcript location", canonical_transcript_location, "Number of exon or intron variant is located in canonical transcript, out of total"), - # ONLY FOR variant.transcript.csq_canonical=true - ("Canonical transcript coding effect", canonical_transcript_consequence_display_title, "Coding effect of variant in canonical transcript"), - # ONLY FOR variant.transcript.csq_most_severe=true - ("Most severe transcript ID", most_severe_transcript_csq_feature, "Ensembl ID of transcript with worst annotation for variant"), - # ONLY FOR variant.transcript.csq_most_severe=true; use csq_intron if csq_exon not present (display as in annotation space: eg. exon 34/45 or intron 4/7) - ("Most severe transcript location", most_severe_transcript_location, "Number of exon or intron variant is located in most severe transcript, out of total"), - # ONLY FOR variant.transcript.csq_most_severe=true - ("Most severe transcript coding effect", most_severe_transcript_consequence_display_title, "Coding effect of variant in most severe transcript"), - ("Inheritance modes", "inheritance_modes", "Inheritance Modes of variant"), - ("NovoPP", "novoPP", "Novocaller Posterior Probability"), - ("Cmphet mate", "cmphet.comhet_mate_variant", "Variant ID of mate, if variant is part of a compound heterozygous group"), - ("Variant Quality", "QUAL", "Variant call quality score"), - ("Genotype Quality", "GQ", "Genotype call quality score"), - ("Strand Bias", "FS", "Strand bias estimated using Fisher's exact test"), - ("Allele Depth", "AD_ALT", "Number of reads with variant allele"), - ("Read Depth", "DP", "Total number of reads at position"), - ("clinvar ID", "variant.csq_clinvar", "Clinvar ID of variant"), - ("gnomADv3 total AF", "variant.csq_gnomadg_af", "Total allele frequency in gnomad v3 (genomes)"), - ("gnomADv3 popmax AF", "variant.csq_gnomadg_af_popmax", "Max. allele frequency in gnomad v3 (genomes)"), - # Name of population where `csq_gnomadg_af-<***> == csq_gnomadg_af_popmax`; use name in title (e.g. African-American/African) - ("gnomADv3 popmax population", gnomadv3_popmax_population, "Population with max. allele frequency in gnomad v3 (genomes)"), - ("gnomADv2 exome total AF", "variant.csq_gnomade2_af", "Total allele frequency in gnomad v2 (exomes)"), - ("gnomADv2 exome popmax AF", "variant.csq_gnomade2_af_popmax", "Max. allele frequency in gnomad v2 (exomes)"), - # Name of population where `csq_gnomade2_af-<***> == csq_gnomade2_af_popmax`; use name in title (e.g. African-American/African) - ("gnomADv2 exome popmax population", gnomadv2_popmax_population, "Population with max. allele frequency in gnomad v2 (exomes)"), - ("GERP++", "variant.csq_gerp_rs", "GERP++ score"), - ("CADD", "variant.csq_cadd_phred", "CADD score"), - ("phyloP-30M", "variant.csq_phylop30way_mammalian", "phyloP (30 Mammals) score"), - ("phyloP-100V", "variant.csq_phylop100way_vertebrate", "phyloP (100 Vertebrates) score"), - ("phastCons-100V", "variant.csq_phastcons100way_vertebrate", "phastCons (100 Vertebrates) score"), - ("SIFT", "variant.csq_sift_pred", "SIFT prediction"), - ("PolyPhen2", "variant.csq_polyphen2_hvar_pred", "PolyPhen2 prediction"), - ("PrimateAI", "variant.csq_primateai_pred", "Primate AI prediction"), - ("REVEL", "variant.csq_revel_score", "REVEL score"), - ("SpliceAI", "variant.spliceaiMaxds", "SpliceAI score"), - ("LOEUF", "variant.genes.genes_most_severe_gene.oe_lof_upper", "Loss-of-function observed/expected upper bound fraction"), - ("RVIS (ExAC)", "variant.genes.genes_most_severe_gene.rvis_exac", "RVIS (Residual Variation Intolerance Score) genome-wide percentile from ExAC"), - ("S-het", "variant.genes.genes_most_severe_gene.s_het", "Estimates of heterozygous selection (source: Cassa et al 2017 Nat Genet doi:10.1038/ng.3831)"), - ("MaxEntScan", "variant.genes.genes_most_severe_maxentscan_diff", "Difference in MaxEntScan scores (Maximum Entropy based scores of splicing strength) between Alt and Ref alleles"), - ("ACMG classification (curr)", "interpretation.classification", "ACMG classification for variant in this case"), - ("ACMG rules (curr)", "interpretation.acmg_rules_invoked.acmg_rule_name", "ACMG rules invoked for variant in this case"), - ("Clinical interpretation notes (curr)", "interpretation.note_text", "Clinical interpretation notes written for this case"), - ("Gene candidacy (curr)", "discovery_interpretation.gene_candidacy", "Gene candidacy level selected for this case"), - ("Variant candidacy (curr)", "discovery_interpretation.variant_candidacy", "Variant candidacy level selected for this case"), - ("Discovery notes (curr)", "discovery_interpretation.note_text", "Gene/variant discovery notes written for this case"), - ("Variant notes (curr)", "variant_notes.note_text", "Additional notes on variant written for this case"), - ("Gene notes (curr)", "gene_notes.note_text", "Additional notes on gene written for this case"), - # For next 6, grab only from note from same project as the VariantSample - ("ACMG classification (prev)", own_project_note_factory("variant.interpretations", "classification"), "ACMG classification for variant in previous cases"), - ("ACMG rules (prev)", own_project_note_factory("variant.interpretations", "acmg"), "ACMG rules invoked for variant in previous cases"), - ("Clinical interpretation (prev)", own_project_note_factory("variant.interpretations", "note_text"), "Clinical interpretation notes written for previous cases"), - ("Gene candidacy (prev)", own_project_note_factory("variant.discovery_interpretations", "gene_candidacy"), "Gene candidacy level selected for previous cases"), - ("Variant candidacy (prev)", own_project_note_factory("variant.discovery_interpretations", "variant_candidacy"), "Variant candidacy level selected for previous cases"), - ("Discovery notes (prev)", own_project_note_factory("variant.discovery_interpretations", "note_text"), "Gene/variant discovery notes written for previous cases"), - ("Variant notes (prev)", own_project_note_factory("variant.variant_notes", "note_text"), "Additional notes on variant written for previous cases"), - ("Gene notes (prev)", own_project_note_factory("variant.genes.genes_most_severe_gene.gene_notes", "note_text"), "Additional notes on gene written for previous cases"), - ] - - -def get_fields_to_embed(spreadsheet_mappings): - fields_to_embed = [ - ## Most of these are needed for columns with render/transform/custom-logic functions in place of (string) CGAP field. - ## Keep up-to-date with any custom logic. - "@id", - "@type", - "project", # Used to get most recent notes of same project from Variant & Gene - "variant.transcript.csq_canonical", - "variant.transcript.csq_most_severe", - "variant.transcript.csq_feature", - "variant.transcript.csq_consequence.impact", - "variant.transcript.csq_consequence.var_conseq_name", - "variant.transcript.csq_consequence.display_title", - "variant.transcript.csq_exon", - "variant.transcript.csq_intron", - ## Notes (e.g. as used by `own_project_note_factory`) - "variant.interpretations.classification", - "variant.interpretations.acmg", - "variant.interpretations.note_text", - "variant.interpretations.project", # @id (string) form - "variant.discovery_interpretations.gene_candidacy", - "variant.discovery_interpretations.variant_candidacy", - "variant.discovery_interpretations.note_text", - "variant.discovery_interpretations.project", # @id (string) form - "variant.variant_notes.note_text", - "variant.variant_notes.project", # @id (string) form - "variant.genes.genes_most_severe_gene.gene_notes.note_text", - "variant.genes.genes_most_severe_gene.gene_notes.project" + spreadsheet_column.get_evaluator() + for spreadsheet_column in spreadsheet_columns + if spreadsheet_column.is_property_evaluator() ] - for pop_suffix, pop_name in POPULATION_SUFFIX_TITLE_TUPLES: - fields_to_embed.append("variant.csq_gnomadg_af-" + pop_suffix) - fields_to_embed.append("variant.csq_gnomade2_af-" + pop_suffix) - for column_title, cgap_field_or_func, description in spreadsheet_mappings: - if isinstance(cgap_field_or_func, str): - # We don't expect any duplicate fields (else would've used a set in place of list) ... pls avoid duplicates in spreadsheet_mappings. - fields_to_embed.append(cgap_field_or_func) - return fields_to_embed diff --git a/src/encoded/util.py b/src/encoded/util.py index 7f477dd646..8fa17f63d4 100644 --- a/src/encoded/util.py +++ b/src/encoded/util.py @@ -396,8 +396,9 @@ def check_user_is_logged_in(request): # # -kmp 1-Sep-2020 +APPLICATION_FORM_ENCODED_MIME_TYPE = "application/x-www-form-urlencoded" CONTENT_TYPE_SPECIAL_CASES = { - 'application/x-www-form-urlencoded': [ + APPLICATION_FORM_ENCODED_MIME_TYPE: [ # Single legacy special case to allow us to POST to metadata TSV requests via form submission. # All other special case values should be added using register_path_content_type. '/metadata/', @@ -606,3 +607,11 @@ def transfer_properties(source, target, properties, property_replacements=None): if property_replacements: property_name = property_replacements.get(property_name, property_name) target[property_name] = property_value + + +def snake_case_to_kebab_case(string: str) -> str: + return string.replace("_", "-") + + +def format_to_url(string: str) -> str: + return f"/{snake_case_to_kebab_case(string)}/"