diff --git a/.gitignore b/.gitignore index abd32e8..db4cbd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*__pycache__* \ No newline at end of file +*__pycache__* +.ipynb_checkpoints \ No newline at end of file diff --git a/Tutorial.ipynb b/Tutorial.ipynb index 96d0203..1e100b8 100644 --- a/Tutorial.ipynb +++ b/Tutorial.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 32, + "execution_count": 13, "id": "fa7b9741", "metadata": {}, "outputs": [], @@ -20,18 +20,22 @@ "# app_id = \"iEyMhfNc3t2chP3LpeiafCtrDSj5AWoYpnRbj0lY\"\n", "# api_path = \"http://localhost:8000/api/v1\"\n", "\n", + "client_id = \"9vEEtvmA3SwmKh0D4nz6e55U0xVZ1io73hEJWy4C\"\n", + "app_id = \"uN4NNFzR1rB5WHhbNIjEtpDID9xWhP894NdQF3K3\"\n", + "api_path = \"http://localhost:8000/api/v1\"\n", + "\n", "headers = {\n", " \"x-client-id\": client_id,\n", " \"x-application-id\": app_id,\n", " \"Content-Type\": \"application/json\",\n", "}\n", "\n", - "metadata = {\"metadata\": json.load(open(\"example-hdruk212.json\"))}\n" + "metadata = {\"metadata\": json.load(open(\"example-hdruk300.json\"))}\n" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 14, "id": "d28497d4", "metadata": {}, "outputs": [ @@ -41,8 +45,8 @@ "text": [ "{\n", " \"message\": \"created\",\n", - " \"data\": 865,\n", - " \"version\": 869\n", + " \"data\": 852,\n", + " \"version\": 852\n", "}\n" ] } @@ -56,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 15, "id": "8c59364c", "metadata": {}, "outputs": [ @@ -66,7 +70,7 @@ "201" ] }, - "execution_count": 34, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -77,17 +81,17 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 16, "id": "aeeebe94", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "865" + "852" ] }, - "execution_count": 35, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -99,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 17, "id": "193193ac", "metadata": {}, "outputs": [], @@ -112,17 +116,17 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 18, "id": "e65a292e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "865" + "852" ] }, - "execution_count": 37, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -133,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 19, "id": "f048faa2", "metadata": {}, "outputs": [ @@ -156,7 +160,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 20, "id": "887851fc", "metadata": {}, "outputs": [ @@ -169,17 +173,18 @@ " \"contactPoint\": \"enquiries@nhsdigital.nhs.uk\",\n", " \"keywords\": \"Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS\",\n", " \"controlledKeywords\": null,\n", - " \"datasetType\": \"Healthdata\",\n", + " \"datasetType\": \"Health and disease\",\n", + " \"datasetSubType\": \"Not applicable\",\n", + " \"populationSize\": -1,\n", " \"description\": \"Collecting information about people in contact with adult psychological therapy services in England. The IAPT data set was developed with the IAPT programme as a patient level, output based, secondary uses data set which aims to deliver robust, comprehensive, nationally consistent and comparable information for patients accessing NHS-funded IAPT services in England. This national data set has been collected since April 2012 and is a mandatory submission for all NHS funded care, including care delivered by independent sector healthcare providers. Data collection on patients with depression and anxiety disorders that are offered psychological therapies, so that we can improve the delivery of care for these conditions.\\n\\nProviders of NHS-funded IAPT services are required to submit data to NHS Digital on a monthly basis.\\n\\nAs a secondary uses data set the IAPT data set re-uses clinical and operational data for purposes other than direct patient care. It defines the data items, definitions and associated value sets extracted or derived from local information systems and sent to NHS Digital for analysis purposes. Timescales for dissemination can be found under 'Our Service Levels' at the following link: https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-process\",\n", " \"doiName\": null,\n", " \"shortTitle\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"title\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"publisher\": {\n", - " \"gatewayId\": \"9e8f9c6f-373c-411e-be03-12c2d492e535\",\n", - " \"name\": \"Conn PLC\"\n", + " \"gatewayId\": \"73bd39bf-f63f-4621-a3d9-be57b75a768e\",\n", + " \"name\": \"Kuhic PLC\"\n", " },\n", - " \"populationSize\": -1,\n", - " \"datasetSubType\": null\n", + " \"inPipeline\": null\n", "}\n" ] } @@ -190,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 21, "id": "8f232012", "metadata": {}, "outputs": [ @@ -200,7 +205,7 @@ "['3.0.0']" ] }, - "execution_count": 40, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -213,17 +218,17 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 22, "id": "a2d3b278", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[869]" + "[852]" ] }, - "execution_count": 41, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -234,7 +239,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 23, "id": "591b9b24", "metadata": {}, "outputs": [ @@ -246,28 +251,27 @@ " \"pathway\": null,\n", " \"spatial\": \"United Kingdom,England\",\n", " \"followup\": \"0 - 6 MONTHS\",\n", - " \"typicalAgeRange\": \"18-150\",\n", - " \"biologicalsamples\": [],\n", + " \"datasetCompleteness\": null,\n", " \"gender\": [],\n", - " \"psychological\": [],\n", - " \"physical\": [],\n", - " \"anthropometric\": [],\n", - " \"lifestyle\": [],\n", - " \"socioeconomic\": []\n", + " \"typicalAgeRangeMin\": 18,\n", + " \"typicalAgeRangeMax\": 150,\n", + " \"materialType\": [\n", + " \"None/not available\"\n", + " ]\n", "}\n" ] } ], "source": [ "response = requests.get(\n", - " f\"{api_path}/integrations/datasets/{dataset_id}?schema_model=HDRUK&schema_version=2.2.1\", headers=headers\n", + " f\"{api_path}/integrations/datasets/{dataset_id}?schema_model=HDRUK&schema_version=3.0.0\", headers=headers\n", ")\n", "print (json.dumps(response.json()['data']['coverage'],indent=6))" ] }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 24, "id": "a0874904", "metadata": {}, "outputs": [ @@ -298,12 +302,12 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 26, "id": "9ae97f85", "metadata": {}, "outputs": [], "source": [ - "temp = json.load(open(\"example-hdruk212.json\"))\n", + "temp = json.load(open(\"example-hdruk300.json\"))\n", "temp['version'] = '3.0.1'\n", "temp['summary']['title'] = 'UPDATED TITLE'\n", "\n", @@ -312,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 27, "id": "14be6979", "metadata": {}, "outputs": [ @@ -323,12 +327,12 @@ "{\n", " \"message\": \"success\",\n", " \"data\": {\n", - " \"id\": 865,\n", + " \"id\": 852,\n", " \"mongo_object_id\": null,\n", " \"mongo_id\": null,\n", " \"mongo_pid\": null,\n", " \"datasetid\": null,\n", - " \"pid\": \"d530e0f1-5da5-47b6-bb7b-fe3cda54db26\",\n", + " \"pid\": \"13aa22ae-6e08-49fa-a23e-fbd5b08c6d3a\",\n", " \"source\": null,\n", " \"discourse_topic_id\": 0,\n", " \"is_cohort_discovery\": false,\n", @@ -336,34 +340,34 @@ " \"state_id\": 0,\n", " \"uploader_id\": 0,\n", " \"metadataquality_id\": 0,\n", - " \"user_id\": 3,\n", - " \"team_id\": 1,\n", + " \"user_id\": 12,\n", + " \"team_id\": 3,\n", " \"views_count\": 0,\n", " \"views_prev_count\": 0,\n", " \"has_technical_details\": 1,\n", - " \"created\": \"2024-04-09 13:10:13\",\n", - " \"updated\": \"2024-04-09 13:10:49\",\n", - " \"submitted\": \"2024-04-09 13:10:13\",\n", + " \"created\": \"2024-06-21 10:34:27\",\n", + " \"updated\": \"2024-06-21 14:29:27\",\n", + " \"submitted\": \"2024-06-21 10:34:27\",\n", " \"published\": null,\n", - " \"created_at\": \"2024-04-09T13:10:13.000000Z\",\n", - " \"updated_at\": \"2024-04-09T13:10:49.000000Z\",\n", + " \"created_at\": \"2024-06-21T10:34:27.000000Z\",\n", + " \"updated_at\": \"2024-06-21T14:29:27.000000Z\",\n", " \"deleted_at\": null,\n", " \"create_origin\": \"API\",\n", " \"status\": \"ACTIVE\",\n", " \"versions\": [\n", " {\n", - " \"id\": 869,\n", - " \"created_at\": \"2024-04-09T13:10:13.000000Z\",\n", - " \"updated_at\": \"2024-04-09T13:10:13.000000Z\",\n", + " \"id\": 852,\n", + " \"created_at\": \"2024-06-21T10:34:27.000000Z\",\n", + " \"updated_at\": \"2024-06-21T10:34:27.000000Z\",\n", " \"deleted_at\": null,\n", - " \"dataset_id\": 865,\n", + " \"dataset_id\": 852,\n", " \"metadata\": {\n", " \"metadata\": {\n", " \"required\": {\n", - " \"gatewayId\": \"865\",\n", - " \"gatewayPid\": \"d530e0f1-5da5-47b6-bb7b-fe3cda54db26\",\n", - " \"issued\": \"2024-04-09T13:10:13.704312Z\",\n", - " \"modified\": \"2024-04-09T13:10:13.704317Z\",\n", + " \"gatewayId\": \"852\",\n", + " \"gatewayPid\": \"13aa22ae-6e08-49fa-a23e-fbd5b08c6d3a\",\n", + " \"issued\": \"2024-06-21T10:34:27.690741Z\",\n", + " \"modified\": \"2024-06-21T10:34:27.690745Z\",\n", " \"revisions\": [],\n", " \"version\": \"3.0.0\"\n", " },\n", @@ -372,36 +376,32 @@ " \"contactPoint\": \"enquiries@nhsdigital.nhs.uk\",\n", " \"keywords\": \"Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS\",\n", " \"controlledKeywords\": null,\n", - " \"datasetType\": \"Healthdata\",\n", + " \"datasetType\": \"Health and disease\",\n", + " \"datasetSubType\": \"Not applicable\",\n", + " \"populationSize\": -1,\n", " \"description\": \"Collecting information about people in contact with adult psychological therapy services in England. The IAPT data set was developed with the IAPT programme as a patient level, output based, secondary uses data set which aims to deliver robust, comprehensive, nationally consistent and comparable information for patients accessing NHS-funded IAPT services in England. This national data set has been collected since April 2012 and is a mandatory submission for all NHS funded care, including care delivered by independent sector healthcare providers. Data collection on patients with depression and anxiety disorders that are offered psychological therapies, so that we can improve the delivery of care for these conditions.\\n\\nProviders of NHS-funded IAPT services are required to submit data to NHS Digital on a monthly basis.\\n\\nAs a secondary uses data set the IAPT data set re-uses clinical and operational data for purposes other than direct patient care. It defines the data items, definitions and associated value sets extracted or derived from local information systems and sent to NHS Digital for analysis purposes. Timescales for dissemination can be found under 'Our Service Levels' at the following link: https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-process\",\n", " \"doiName\": null,\n", " \"shortTitle\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"title\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"publisher\": {\n", - " \"gatewayId\": \"9e8f9c6f-373c-411e-be03-12c2d492e535\",\n", - " \"name\": \"Conn PLC\"\n", + " \"gatewayId\": \"73bd39bf-f63f-4621-a3d9-be57b75a768e\",\n", + " \"name\": \"Kuhic PLC\"\n", " },\n", - " \"populationSize\": -1,\n", - " \"datasetSubType\": null\n", + " \"inPipeline\": null\n", " },\n", " \"coverage\": {\n", " \"pathway\": null,\n", " \"spatial\": \"United Kingdom,England\",\n", " \"followup\": \"0 - 6 MONTHS\",\n", " \"typicalAgeRange\": \"18-150\",\n", - " \"biologicalsamples\": null,\n", - " \"gender\": null,\n", - " \"psychological\": null,\n", - " \"physical\": null,\n", - " \"anthropometric\": null,\n", - " \"lifestyle\": null,\n", - " \"socioeconomic\": null\n", + " \"datasetCompleteness\": null\n", " },\n", " \"provenance\": {\n", " \"origin\": {\n", - " \"purpose\": \"OTHER,ADMINISTRATIVE\",\n", + " \"purpose\": \"Other,Administrative\",\n", " \"source\": \"EPR\",\n", - " \"collectionSituation\": \"COMMUNITY,PRIMARY CARE\"\n", + " \"collectionSituation\": null,\n", + " \"imageContrast\": null\n", " },\n", " \"temporal\": {\n", " \"endDate\": null,\n", @@ -420,13 +420,14 @@ " \"accessRights\": \"https://digital.nhs.uk/services/data-access-request-service-dars\",\n", " \"accessService\": \"Once your DARS application has been approved, data will be made available either by secure file transfer or through the Data Access Environment (DAE). BL\\n\\nSecure file transfer: https://digital.nhs.uk/services/transfer-data-securely\\n\\nDAE: https://digital.nhs.uk/services/data-access-environment-dae\",\n", " \"accessRequestCost\": \"https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-charges\",\n", - " \"accessServiceCategory\": null\n", + " \"accessServiceCategory\": null,\n", + " \"accessMode\": \"New project\"\n", " },\n", " \"usage\": {\n", - " \"dataUseLimitation\": \"NO RESTRICTION\",\n", - " \"dataUseRequirement\": \"INSTITUTION SPECIFIC RESTRICTIONS,PROJECT SPECIFIC RESTRICTIONS,TIME LIMIT ON USE\",\n", + " \"dataUseLimitation\": \"No restriction\",\n", + " \"dataUseRequirement\": null,\n", " \"resourceCreator\": {\n", - " \"name\": \"NHS DIGITIAL\",\n", + " \"name\": null,\n", " \"gatewayId\": null,\n", " \"rorId\": null\n", " }\n", @@ -439,37 +440,39 @@ " }\n", " },\n", " \"linkage\": {\n", - " \"associatedMedia\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set,https://digital.nhs.uk/about-nhs-digital/corporate-information-and-documents/directions-and-data-provision-notices/data-provision-notices-dpns/improving-access-to-psychological-therapies-data-set-data-provision-notice\",\n", + " \"isGeneratedUsing\": null,\n", + " \"associatedMedia\": null,\n", + " \"dataUses\": null,\n", " \"isReferenceIn\": null,\n", - " \"tools\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set/submitting-iapt-data\",\n", + " \"tools\": null,\n", " \"datasetLinkage\": {\n", - " \"isDerivedFrom\": \"Data will be minimised as appropriate relative to the data access application\",\n", - " \"isPartOf\": \"mental health\",\n", - " \"linkedDatasets\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"isMemberOf\": null\n", + " \"isDerivedFrom\": null,\n", + " \"isPartOf\": null,\n", + " \"isMemberOf\": null,\n", + " \"linkedDatasets\": null\n", " },\n", - " \"investigations\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"isGeneratedUsing\": null,\n", - " \"dataUses\": null,\n", - " \"syntheticDataWebLink\": null\n", + " \"investigations\": null,\n", + " \"syntheticDataWebLink\": null,\n", + " \"publicationAboutDataset\": null,\n", + " \"publicationUsingDataset\": null\n", " },\n", " \"observations\": [\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 114125,\n", " \"observationDate\": \"2021-02-25\",\n", " \"measuredProperty\": \"COUNT\",\n", " \"disambiguatingDescription\": \"In December 2020 there were 114,125 referrals made into IAPT services in England.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 81578,\n", " \"observationDate\": \"2021-02-25\",\n", " \"measuredProperty\": \"COUNT\",\n", " \"disambiguatingDescription\": \"In December 2020, 81,578 referrals into IAPT services in England started a course of treatment.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 48331,\n", " \"observationDate\": \"2021-02-25\",\n", " \"measuredProperty\": \"COUNT\",\n", @@ -486,13 +489,7 @@ " \"description\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", " \"dataType\": \"Number\",\n", " \"sensitive\": false,\n", - " \"values\": [\n", - " {\n", - " \"name\": \"Male\",\n", - " \"frequency\": 50,\n", - " \"description\": null\n", - " }\n", - " ]\n", + " \"values\": null\n", " }\n", " ]\n", " },\n", @@ -505,18 +502,11 @@ " \"description\": \"A request for the provision of care services to a PATIENT.\",\n", " \"dataType\": \"String\",\n", " \"sensitive\": false,\n", - " \"values\": [\n", - " {\n", - " \"name\": \"Male\",\n", - " \"frequency\": 50,\n", - " \"description\": null\n", - " }\n", - " ]\n", + " \"values\": null\n", " }\n", " ]\n", " }\n", - " ],\n", - " \"tissuesSampleCollection\": null\n", + " ]\n", " },\n", " \"app\": {\n", " \"id\": 0\n", @@ -530,52 +520,66 @@ " \"summary\": {\n", " \"title\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"abstract\": \"Patient-level data set that captures information about people in contact with services commissioned as part of the adult Improving Access to Psychological Services (IAPT) programme.\",\n", - " \"publisher\": {\n", + " \"contactPoint\": \"enquiries@nhsdigital.nhs.uk\",\n", + " \"keywords\": \"Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS\",\n", + " \"alternateIdentifiers\": null,\n", + " \"doiName\": null,\n", + " \"populationSize\": -1,\n", + " \"dataProvider\": {\n", " \"identifier\": \"https://web.www.healthdatagateway.org/5f86cd34980f41c6f02261f4\",\n", " \"name\": \"NHS DIGITAL\",\n", " \"logo\": null,\n", " \"description\": null,\n", " \"contactPoint\": null,\n", " \"memberOf\": \"ALLIANCE\"\n", - " },\n", - " \"contactPoint\": \"enquiries@nhsdigital.nhs.uk\",\n", - " \"keywords\": \"Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS\",\n", - " \"alternateIdentifiers\": null,\n", - " \"doiName\": null\n", + " }\n", " },\n", " \"documentation\": {\n", " \"description\": \"Collecting information about people in contact with adult psychological therapy services in England. The IAPT data set was developed with the IAPT programme as a patient level, output based, secondary uses data set which aims to deliver robust, comprehensive, nationally consistent and comparable information for patients accessing NHS-funded IAPT services in England. This national data set has been collected since April 2012 and is a mandatory submission for all NHS funded care, including care delivered by independent sector healthcare providers. Data collection on patients with depression and anxiety disorders that are offered psychological therapies, so that we can improve the delivery of care for these conditions.\\n\\nProviders of NHS-funded IAPT services are required to submit data to NHS Digital on a monthly basis.\\n\\nAs a secondary uses data set the IAPT data set re-uses clinical and operational data for purposes other than direct patient care. It defines the data items, definitions and associated value sets extracted or derived from local information systems and sent to NHS Digital for analysis purposes. Timescales for dissemination can be found under 'Our Service Levels' at the following link: https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-process\",\n", - " \"associatedMedia\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set,https://digital.nhs.uk/about-nhs-digital/corporate-information-and-documents/directions-and-data-provision-notices/data-provision-notices-dpns/improving-access-to-psychological-therapies-data-set-data-provision-notice\",\n", - " \"isPartOf\": \"mental health\"\n", + " \"associatedMedia\": null,\n", + " \"inPipeline\": null\n", " },\n", " \"coverage\": {\n", " \"spatial\": \"United Kingdom,England\",\n", - " \"typicalAgeRange\": \"18-150\",\n", - " \"physicalSampleAvailability\": null,\n", " \"followup\": \"0 - 6 MONTHS\",\n", - " \"pathway\": null\n", + " \"pathway\": null,\n", + " \"gender\": null,\n", + " \"typicalAgeRangeMin\": 18,\n", + " \"typicalAgeRangeMax\": 150,\n", + " \"datasetCompleteness\": null,\n", + " \"materialType\": [\n", + " \"None/not available\"\n", + " ]\n", " },\n", " \"provenance\": {\n", " \"origin\": {\n", - " \"purpose\": \"OTHER,ADMINISTRATIVE\",\n", - " \"source\": \"EPR\",\n", - " \"collectionSituation\": \"COMMUNITY,PRIMARY CARE\"\n", + " \"purpose\": [\n", + " \"Other\",\n", + " \"Administrative\"\n", + " ],\n", + " \"source\": [\n", + " \"EPR\"\n", + " ],\n", + " \"datasetType\": \"Health and disease\",\n", + " \"datasetSubType\": \"Not applicable\",\n", + " \"collectionSource\": null,\n", + " \"imageContrast\": null\n", " },\n", " \"temporal\": {\n", - " \"accrualPeriodicity\": \"MONTHLY\",\n", " \"distributionReleaseDate\": null,\n", " \"startDate\": \"2012-04-01\",\n", " \"endDate\": null,\n", - " \"timeLag\": \"2-6 MONTHS\"\n", + " \"timeLag\": \"2-6 months\",\n", + " \"publishingFrequency\": \"Monthly\"\n", " }\n", " },\n", " \"accessibility\": {\n", " \"usage\": {\n", - " \"dataUseLimitation\": \"NO RESTRICTION\",\n", - " \"dataUseRequirements\": \"INSTITUTION SPECIFIC RESTRICTIONS,PROJECT SPECIFIC RESTRICTIONS,TIME LIMIT ON USE\",\n", - " \"resourceCreator\": \"NHS DIGITIAL\",\n", - " \"investigations\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"isReferencedBy\": null\n", + " \"dataUseLimitation\": [\n", + " \"No restriction\"\n", + " ],\n", + " \"dataUseRequirements\": null,\n", + " \"resourceCreator\": null\n", " },\n", " \"access\": {\n", " \"accessRights\": \"https://digital.nhs.uk/services/data-access-request-service-dars\",\n", @@ -584,131 +588,148 @@ " \"deliveryLeadTime\": null,\n", " \"jurisdiction\": \"GB-ENG\",\n", " \"dataProcessor\": null,\n", - " \"dataController\": \"NHS DIGITAL\"\n", + " \"dataController\": \"NHS DIGITAL\",\n", + " \"accessServiceCategory\": null,\n", + " \"accessMode\": \"New project\"\n", " },\n", " \"formatAndStandards\": {\n", - " \"vocabularyEncodingScheme\": \"ODS,SNOMED CT,NHS NATIONAL CODES,ICD10\",\n", - " \"conformsTo\": \"NHS DATA DICTIONARY\",\n", + " \"vocabularyEncodingScheme\": [\n", + " \"ODS\",\n", + " \"SNOMED CT\",\n", + " \"NHS NATIONAL CODES\",\n", + " \"ICD10\"\n", + " ],\n", + " \"conformsTo\": [\n", + " \"NHS DATA DICTIONARY\"\n", + " ],\n", " \"language\": \"en\",\n", " \"format\": \"CSV\"\n", " }\n", " },\n", " \"enrichmentAndLinkage\": {\n", - " \"qualifiedRelation\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"derivation\": \"Data will be minimised as appropriate relative to the data access application\",\n", - " \"tools\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set/submitting-iapt-data\"\n", + " \"tools\": null,\n", + " \"derivedFrom\": null,\n", + " \"isPartOf\": null,\n", + " \"linkableDatasets\": null,\n", + " \"similarToDatasets\": null,\n", + " \"publicationAboutDataset\": null,\n", + " \"investigations\": null,\n", + " \"publicationUsingDataset\": null\n", " },\n", " \"observations\": [\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 114125,\n", " \"measuredProperty\": \"COUNT\",\n", " \"observationDate\": \"2021-02-25\",\n", " \"disambiguatingDescription\": \"In December 2020 there were 114,125 referrals made into IAPT services in England.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 81578,\n", " \"measuredProperty\": \"COUNT\",\n", " \"observationDate\": \"2021-02-25\",\n", " \"disambiguatingDescription\": \"In December 2020, 81,578 referrals into IAPT services in England started a course of treatment.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 48331,\n", " \"measuredProperty\": \"COUNT\",\n", " \"observationDate\": \"2021-02-25\",\n", " \"disambiguatingDescription\": \"In December 2020, 48,331 referrals into IAPT services in England completed a course of treatment.\"\n", " }\n", " ],\n", - " \"structuralMetadata\": [\n", - " {\n", - " \"name\": \"IAPT.iapt.Rep_Referral\",\n", - " \"description\": \"IAPT.iapt.Rep_Referral\",\n", - " \"elements\": [\n", - " {\n", - " \"name\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", - " \"description\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", - " \"dataType\": \"Number\",\n", - " \"sensitive\": false\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"name\": \"IAPT.iapt.Rep_Referral\",\n", - " \"description\": \"IAPT.iapt.Rep_Referral\",\n", - " \"elements\": [\n", - " {\n", - " \"name\": \"Pseudonymised Service Request Identifier\",\n", - " \"description\": \"A request for the provision of care services to a PATIENT.\",\n", - " \"dataType\": \"String\",\n", - " \"sensitive\": false\n", - " }\n", - " ]\n", - " }\n", - " ]\n", + " \"structuralMetadata\": {\n", + " \"tables\": [\n", + " {\n", + " \"name\": \"IAPT.iapt.Rep_Referral\",\n", + " \"description\": \"IAPT.iapt.Rep_Referral\",\n", + " \"columns\": [\n", + " {\n", + " \"name\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", + " \"description\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", + " \"dataType\": \"Number\",\n", + " \"sensitive\": false,\n", + " \"values\": null\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"name\": \"IAPT.iapt.Rep_Referral\",\n", + " \"description\": \"IAPT.iapt.Rep_Referral\",\n", + " \"columns\": [\n", + " {\n", + " \"name\": \"Pseudonymised Service Request Identifier\",\n", + " \"description\": \"A request for the provision of care services to a PATIENT.\",\n", + " \"dataType\": \"String\",\n", + " \"sensitive\": false,\n", + " \"values\": null\n", + " }\n", + " ]\n", + " }\n", + " ],\n", + " \"syntheticDataWebLink\": null\n", + " }\n", " },\n", - " \"gwdmVersion\": \"1.2\"\n", + " \"gwdmVersion\": \"2.0\"\n", " },\n", - " \"version\": 1\n", + " \"version\": 1,\n", + " \"provider_team_id\": null,\n", + " \"application_type\": null\n", " },\n", " {\n", - " \"id\": 870,\n", - " \"created_at\": \"2024-04-09T13:10:49.000000Z\",\n", - " \"updated_at\": \"2024-04-09T13:10:49.000000Z\",\n", + " \"id\": 853,\n", + " \"created_at\": \"2024-06-21T14:29:27.000000Z\",\n", + " \"updated_at\": \"2024-06-21T14:29:27.000000Z\",\n", " \"deleted_at\": null,\n", - " \"dataset_id\": 865,\n", + " \"dataset_id\": 852,\n", " \"metadata\": {\n", " \"metadata\": {\n", " \"required\": {\n", - " \"gatewayId\": \"865\",\n", - " \"gatewayPid\": \"d530e0f1-5da5-47b6-bb7b-fe3cda54db26\",\n", + " \"gatewayId\": \"852\",\n", + " \"gatewayPid\": \"13aa22ae-6e08-49fa-a23e-fbd5b08c6d3a\",\n", " \"issued\": \"2021-05-10T00:00:00.000Z\",\n", - " \"modified\": \"2024-04-09T13:10:49.928091Z\",\n", + " \"modified\": \"2024-06-21T14:29:27.758640Z\",\n", + " \"version\": \"3.0.1\",\n", " \"revisions\": [\n", " {\n", - " \"url\": \"https://placeholder.blah/d530e0f1-5da5-47b6-bb7b-fe3cda54db26?version=3.0.0\",\n", + " \"url\": \"https://placeholder.blah/13aa22ae-6e08-49fa-a23e-fbd5b08c6d3a?version=3.0.0\",\n", " \"version\": \"3.0.0\"\n", " }\n", - " ],\n", - " \"version\": \"3.0.1\"\n", + " ]\n", " },\n", " \"summary\": {\n", " \"abstract\": \"Patient-level data set that captures information about people in contact with services commissioned as part of the adult Improving Access to Psychological Services (IAPT) programme.\",\n", " \"contactPoint\": \"enquiries@nhsdigital.nhs.uk\",\n", " \"keywords\": \"Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS\",\n", " \"controlledKeywords\": null,\n", - " \"datasetType\": \"Healthdata\",\n", + " \"datasetType\": \"Health and disease\",\n", + " \"datasetSubType\": \"Not applicable\",\n", + " \"populationSize\": -1,\n", " \"description\": \"Collecting information about people in contact with adult psychological therapy services in England. The IAPT data set was developed with the IAPT programme as a patient level, output based, secondary uses data set which aims to deliver robust, comprehensive, nationally consistent and comparable information for patients accessing NHS-funded IAPT services in England. This national data set has been collected since April 2012 and is a mandatory submission for all NHS funded care, including care delivered by independent sector healthcare providers. Data collection on patients with depression and anxiety disorders that are offered psychological therapies, so that we can improve the delivery of care for these conditions.\\n\\nProviders of NHS-funded IAPT services are required to submit data to NHS Digital on a monthly basis.\\n\\nAs a secondary uses data set the IAPT data set re-uses clinical and operational data for purposes other than direct patient care. It defines the data items, definitions and associated value sets extracted or derived from local information systems and sent to NHS Digital for analysis purposes. Timescales for dissemination can be found under 'Our Service Levels' at the following link: https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-process\",\n", " \"doiName\": null,\n", " \"shortTitle\": \"UPDATED TITLE\",\n", " \"title\": \"UPDATED TITLE\",\n", " \"publisher\": {\n", - " \"name\": \"Conn PLC\",\n", - " \"gatewayId\": \"9e8f9c6f-373c-411e-be03-12c2d492e535\",\n", + " \"name\": \"Kuhic PLC\",\n", + " \"gatewayId\": \"73bd39bf-f63f-4621-a3d9-be57b75a768e\",\n", " \"rorId\": null\n", " },\n", - " \"populationSize\": -1,\n", - " \"datasetSubType\": null\n", + " \"inPipeline\": null\n", " },\n", " \"coverage\": {\n", " \"pathway\": null,\n", " \"spatial\": \"United Kingdom,England\",\n", " \"followup\": \"0 - 6 MONTHS\",\n", " \"typicalAgeRange\": \"18-150\",\n", - " \"biologicalsamples\": null,\n", - " \"gender\": null,\n", - " \"psychological\": null,\n", - " \"physical\": null,\n", - " \"anthropometric\": null,\n", - " \"lifestyle\": null,\n", - " \"socioeconomic\": null\n", + " \"datasetCompleteness\": null\n", " },\n", " \"provenance\": {\n", " \"origin\": {\n", - " \"purpose\": \"OTHER,ADMINISTRATIVE\",\n", + " \"purpose\": \"Other,Administrative\",\n", " \"source\": \"EPR\",\n", - " \"collectionSituation\": \"COMMUNITY,PRIMARY CARE\"\n", + " \"collectionSituation\": null,\n", + " \"imageContrast\": null\n", " },\n", " \"temporal\": {\n", " \"endDate\": null,\n", @@ -727,13 +748,14 @@ " \"accessRights\": \"https://digital.nhs.uk/services/data-access-request-service-dars\",\n", " \"accessService\": \"Once your DARS application has been approved, data will be made available either by secure file transfer or through the Data Access Environment (DAE). BL\\n\\nSecure file transfer: https://digital.nhs.uk/services/transfer-data-securely\\n\\nDAE: https://digital.nhs.uk/services/data-access-environment-dae\",\n", " \"accessRequestCost\": \"https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-charges\",\n", - " \"accessServiceCategory\": null\n", + " \"accessServiceCategory\": null,\n", + " \"accessMode\": \"New project\"\n", " },\n", " \"usage\": {\n", - " \"dataUseLimitation\": \"NO RESTRICTION\",\n", - " \"dataUseRequirement\": \"INSTITUTION SPECIFIC RESTRICTIONS,PROJECT SPECIFIC RESTRICTIONS,TIME LIMIT ON USE\",\n", + " \"dataUseLimitation\": \"No restriction\",\n", + " \"dataUseRequirement\": null,\n", " \"resourceCreator\": {\n", - " \"name\": \"NHS DIGITIAL\",\n", + " \"name\": null,\n", " \"gatewayId\": null,\n", " \"rorId\": null\n", " }\n", @@ -746,37 +768,39 @@ " }\n", " },\n", " \"linkage\": {\n", - " \"associatedMedia\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set,https://digital.nhs.uk/about-nhs-digital/corporate-information-and-documents/directions-and-data-provision-notices/data-provision-notices-dpns/improving-access-to-psychological-therapies-data-set-data-provision-notice\",\n", + " \"isGeneratedUsing\": null,\n", + " \"associatedMedia\": null,\n", + " \"dataUses\": null,\n", " \"isReferenceIn\": null,\n", - " \"tools\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set/submitting-iapt-data\",\n", + " \"tools\": null,\n", " \"datasetLinkage\": {\n", - " \"isDerivedFrom\": \"Data will be minimised as appropriate relative to the data access application\",\n", - " \"isPartOf\": \"mental health\",\n", - " \"linkedDatasets\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"isMemberOf\": null\n", + " \"isDerivedFrom\": null,\n", + " \"isPartOf\": null,\n", + " \"isMemberOf\": null,\n", + " \"linkedDatasets\": null\n", " },\n", - " \"investigations\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"isGeneratedUsing\": null,\n", - " \"dataUses\": null,\n", - " \"syntheticDataWebLink\": null\n", + " \"investigations\": null,\n", + " \"syntheticDataWebLink\": null,\n", + " \"publicationAboutDataset\": null,\n", + " \"publicationUsingDataset\": null\n", " },\n", " \"observations\": [\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 114125,\n", " \"observationDate\": \"2021-02-25\",\n", " \"measuredProperty\": \"COUNT\",\n", " \"disambiguatingDescription\": \"In December 2020 there were 114,125 referrals made into IAPT services in England.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 81578,\n", " \"observationDate\": \"2021-02-25\",\n", " \"measuredProperty\": \"COUNT\",\n", " \"disambiguatingDescription\": \"In December 2020, 81,578 referrals into IAPT services in England started a course of treatment.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 48331,\n", " \"observationDate\": \"2021-02-25\",\n", " \"measuredProperty\": \"COUNT\",\n", @@ -793,13 +817,7 @@ " \"description\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", " \"dataType\": \"Number\",\n", " \"sensitive\": false,\n", - " \"values\": [\n", - " {\n", - " \"name\": \"Male\",\n", - " \"frequency\": 50,\n", - " \"description\": null\n", - " }\n", - " ]\n", + " \"values\": null\n", " }\n", " ]\n", " },\n", @@ -812,23 +830,16 @@ " \"description\": \"A request for the provision of care services to a PATIENT.\",\n", " \"dataType\": \"String\",\n", " \"sensitive\": false,\n", - " \"values\": [\n", - " {\n", - " \"name\": \"Male\",\n", - " \"frequency\": 50,\n", - " \"description\": null\n", - " }\n", - " ]\n", + " \"values\": null\n", " }\n", " ]\n", " }\n", - " ],\n", - " \"tissuesSampleCollection\": null\n", + " ]\n", " },\n", " \"app\": {\n", " \"id\": 0\n", " },\n", - " \"id\": \"865\",\n", + " \"id\": \"852\",\n", " \"original_metadata\": {\n", " \"identifier\": \"https://web.www.healthdatagateway.org/19525c5f-92ee-41b6-bb79-673624b27bdd\",\n", " \"version\": \"3.0.1\",\n", @@ -838,52 +849,66 @@ " \"summary\": {\n", " \"title\": \"UPDATED TITLE\",\n", " \"abstract\": \"Patient-level data set that captures information about people in contact with services commissioned as part of the adult Improving Access to Psychological Services (IAPT) programme.\",\n", - " \"publisher\": {\n", + " \"contactPoint\": \"enquiries@nhsdigital.nhs.uk\",\n", + " \"keywords\": \"Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS\",\n", + " \"alternateIdentifiers\": null,\n", + " \"doiName\": null,\n", + " \"populationSize\": -1,\n", + " \"dataProvider\": {\n", " \"identifier\": \"https://web.www.healthdatagateway.org/5f86cd34980f41c6f02261f4\",\n", " \"name\": \"NHS DIGITAL\",\n", " \"logo\": null,\n", " \"description\": null,\n", " \"contactPoint\": null,\n", " \"memberOf\": \"ALLIANCE\"\n", - " },\n", - " \"contactPoint\": \"enquiries@nhsdigital.nhs.uk\",\n", - " \"keywords\": \"Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS\",\n", - " \"alternateIdentifiers\": null,\n", - " \"doiName\": null\n", + " }\n", " },\n", " \"documentation\": {\n", " \"description\": \"Collecting information about people in contact with adult psychological therapy services in England. The IAPT data set was developed with the IAPT programme as a patient level, output based, secondary uses data set which aims to deliver robust, comprehensive, nationally consistent and comparable information for patients accessing NHS-funded IAPT services in England. This national data set has been collected since April 2012 and is a mandatory submission for all NHS funded care, including care delivered by independent sector healthcare providers. Data collection on patients with depression and anxiety disorders that are offered psychological therapies, so that we can improve the delivery of care for these conditions.\\n\\nProviders of NHS-funded IAPT services are required to submit data to NHS Digital on a monthly basis.\\n\\nAs a secondary uses data set the IAPT data set re-uses clinical and operational data for purposes other than direct patient care. It defines the data items, definitions and associated value sets extracted or derived from local information systems and sent to NHS Digital for analysis purposes. Timescales for dissemination can be found under 'Our Service Levels' at the following link: https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-process\",\n", - " \"associatedMedia\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set,https://digital.nhs.uk/about-nhs-digital/corporate-information-and-documents/directions-and-data-provision-notices/data-provision-notices-dpns/improving-access-to-psychological-therapies-data-set-data-provision-notice\",\n", - " \"isPartOf\": \"mental health\"\n", + " \"associatedMedia\": null,\n", + " \"inPipeline\": null\n", " },\n", " \"coverage\": {\n", " \"spatial\": \"United Kingdom,England\",\n", - " \"typicalAgeRange\": \"18-150\",\n", - " \"physicalSampleAvailability\": null,\n", " \"followup\": \"0 - 6 MONTHS\",\n", - " \"pathway\": null\n", + " \"pathway\": null,\n", + " \"gender\": null,\n", + " \"typicalAgeRangeMin\": 18,\n", + " \"typicalAgeRangeMax\": 150,\n", + " \"datasetCompleteness\": null,\n", + " \"materialType\": [\n", + " \"None/not available\"\n", + " ]\n", " },\n", " \"provenance\": {\n", " \"origin\": {\n", - " \"purpose\": \"OTHER,ADMINISTRATIVE\",\n", - " \"source\": \"EPR\",\n", - " \"collectionSituation\": \"COMMUNITY,PRIMARY CARE\"\n", + " \"purpose\": [\n", + " \"Other\",\n", + " \"Administrative\"\n", + " ],\n", + " \"source\": [\n", + " \"EPR\"\n", + " ],\n", + " \"datasetType\": \"Health and disease\",\n", + " \"datasetSubType\": \"Not applicable\",\n", + " \"collectionSource\": null,\n", + " \"imageContrast\": null\n", " },\n", " \"temporal\": {\n", - " \"accrualPeriodicity\": \"MONTHLY\",\n", " \"distributionReleaseDate\": null,\n", " \"startDate\": \"2012-04-01\",\n", " \"endDate\": null,\n", - " \"timeLag\": \"2-6 MONTHS\"\n", + " \"timeLag\": \"2-6 months\",\n", + " \"publishingFrequency\": \"Monthly\"\n", " }\n", " },\n", " \"accessibility\": {\n", " \"usage\": {\n", - " \"dataUseLimitation\": \"NO RESTRICTION\",\n", - " \"dataUseRequirements\": \"INSTITUTION SPECIFIC RESTRICTIONS,PROJECT SPECIFIC RESTRICTIONS,TIME LIMIT ON USE\",\n", - " \"resourceCreator\": \"NHS DIGITIAL\",\n", - " \"investigations\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"isReferencedBy\": null\n", + " \"dataUseLimitation\": [\n", + " \"No restriction\"\n", + " ],\n", + " \"dataUseRequirements\": null,\n", + " \"resourceCreator\": null\n", " },\n", " \"access\": {\n", " \"accessRights\": \"https://digital.nhs.uk/services/data-access-request-service-dars\",\n", @@ -892,73 +917,94 @@ " \"deliveryLeadTime\": null,\n", " \"jurisdiction\": \"GB-ENG\",\n", " \"dataProcessor\": null,\n", - " \"dataController\": \"NHS DIGITAL\"\n", + " \"dataController\": \"NHS DIGITAL\",\n", + " \"accessServiceCategory\": null,\n", + " \"accessMode\": \"New project\"\n", " },\n", " \"formatAndStandards\": {\n", - " \"vocabularyEncodingScheme\": \"ODS,SNOMED CT,NHS NATIONAL CODES,ICD10\",\n", - " \"conformsTo\": \"NHS DATA DICTIONARY\",\n", + " \"vocabularyEncodingScheme\": [\n", + " \"ODS\",\n", + " \"SNOMED CT\",\n", + " \"NHS NATIONAL CODES\",\n", + " \"ICD10\"\n", + " ],\n", + " \"conformsTo\": [\n", + " \"NHS DATA DICTIONARY\"\n", + " ],\n", " \"language\": \"en\",\n", " \"format\": \"CSV\"\n", " }\n", " },\n", " \"enrichmentAndLinkage\": {\n", - " \"qualifiedRelation\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", - " \"derivation\": \"Data will be minimised as appropriate relative to the data access application\",\n", - " \"tools\": \"https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set/submitting-iapt-data\"\n", + " \"tools\": null,\n", + " \"derivedFrom\": null,\n", + " \"isPartOf\": null,\n", + " \"linkableDatasets\": null,\n", + " \"similarToDatasets\": null,\n", + " \"publicationAboutDataset\": null,\n", + " \"investigations\": null,\n", + " \"publicationUsingDataset\": null\n", " },\n", " \"observations\": [\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 114125,\n", " \"measuredProperty\": \"COUNT\",\n", " \"observationDate\": \"2021-02-25\",\n", " \"disambiguatingDescription\": \"In December 2020 there were 114,125 referrals made into IAPT services in England.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 81578,\n", " \"measuredProperty\": \"COUNT\",\n", " \"observationDate\": \"2021-02-25\",\n", " \"disambiguatingDescription\": \"In December 2020, 81,578 referrals into IAPT services in England started a course of treatment.\"\n", " },\n", " {\n", - " \"observedNode\": \"EVENTS\",\n", + " \"observedNode\": \"Event\",\n", " \"measuredValue\": 48331,\n", " \"measuredProperty\": \"COUNT\",\n", " \"observationDate\": \"2021-02-25\",\n", " \"disambiguatingDescription\": \"In December 2020, 48,331 referrals into IAPT services in England completed a course of treatment.\"\n", " }\n", " ],\n", - " \"structuralMetadata\": [\n", - " {\n", - " \"name\": \"IAPT.iapt.Rep_Referral\",\n", - " \"description\": \"IAPT.iapt.Rep_Referral\",\n", - " \"elements\": [\n", - " {\n", - " \"name\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", - " \"description\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", - " \"dataType\": \"Number\",\n", - " \"sensitive\": false\n", - " }\n", - " ]\n", - " },\n", - " {\n", - " \"name\": \"IAPT.iapt.Rep_Referral\",\n", - " \"description\": \"IAPT.iapt.Rep_Referral\",\n", - " \"elements\": [\n", - " {\n", - " \"name\": \"Pseudonymised Service Request Identifier\",\n", - " \"description\": \"A request for the provision of care services to a PATIENT.\",\n", - " \"dataType\": \"String\",\n", - " \"sensitive\": false\n", - " }\n", - " ]\n", - " }\n", - " ]\n", + " \"structuralMetadata\": {\n", + " \"tables\": [\n", + " {\n", + " \"name\": \"IAPT.iapt.Rep_Referral\",\n", + " \"description\": \"IAPT.iapt.Rep_Referral\",\n", + " \"columns\": [\n", + " {\n", + " \"name\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", + " \"description\": \"Count of number of Non-guided Self Help (Computer) sessions (derived)\",\n", + " \"dataType\": \"Number\",\n", + " \"sensitive\": false,\n", + " \"values\": null\n", + " }\n", + " ]\n", + " },\n", + " {\n", + " \"name\": \"IAPT.iapt.Rep_Referral\",\n", + " \"description\": \"IAPT.iapt.Rep_Referral\",\n", + " \"columns\": [\n", + " {\n", + " \"name\": \"Pseudonymised Service Request Identifier\",\n", + " \"description\": \"A request for the provision of care services to a PATIENT.\",\n", + " \"dataType\": \"String\",\n", + " \"sensitive\": false,\n", + " \"values\": null\n", + " }\n", + " ]\n", + " }\n", + " ],\n", + " \"syntheticDataWebLink\": null\n", + " }\n", " },\n", - " \"gwdmVersion\": \"1.2\"\n", + " \"gwdmVersion\": \"2.0\"\n", " },\n", - " \"version\": 2\n", + " \"version\": 2,\n", + " \"provider_team_id\": null,\n", + " \"application_type\": null\n", " }\n", " ]\n", " }\n", @@ -975,7 +1021,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 28, "id": "dfce3deb", "metadata": {}, "outputs": [], @@ -987,7 +1033,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 29, "id": "e75312a6", "metadata": {}, "outputs": [ @@ -997,7 +1043,7 @@ "['3.0.0', '3.0.1']" ] }, - "execution_count": 47, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -1012,7 +1058,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 30, "id": "b8544866", "metadata": {}, "outputs": [], @@ -1025,7 +1071,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 31, "id": "ed8402dd", "metadata": {}, "outputs": [ @@ -1035,7 +1081,7 @@ "{'message': 'success'}" ] }, - "execution_count": 49, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } diff --git a/docs/creating-a-dataset.md b/docs/creating-a-dataset.md index 266c8cf..31d66f6 100644 --- a/docs/creating-a-dataset.md +++ b/docs/creating-a-dataset.md @@ -132,12 +132,14 @@ Navigate (on the Gateway) to `Integrations > API Management > Manage API`, and s "2.1.0", "2.0.2", "2.2.1", - "2.2.0" + "2.2.0", + "3.0.0" ], "GWDM": [ "1.0", "1.1", - "1.2" + "1.2", + "2.0" ], "SchemaOrg": [ "default", diff --git a/docs/creating-metadata.md b/docs/creating-metadata.md index b7038c9..c4bce6d 100644 --- a/docs/creating-metadata.md +++ b/docs/creating-metadata.md @@ -2,20 +2,21 @@ There are multiple [metadata schemas](https://hdruk.github.io/schemata-2/) that !!! tip - - **Recommended:** [HDRUK 2.2.0](https://hdruk.github.io/schemata-2/HDRUK/2.2.0/) is our public facing schema. Our manual-onboarding forms are based on this schema. + - **Recommended:** [HDRUK 3.0.0](https://hdruk.github.io/schemata-2/HDRUK/3.0.0/) is our public facing schema. Our manual-onboarding forms are based on this schema. - Older supported versions: + - [HDRUK 2.2.1](https://hdruk.github.io/schemata-2/HDRUK/2.2.1/) - [HDRUK 2.1.2](https://hdruk.github.io/schemata-2/HDRUK/2.1.2/) - [HDRUK 2.1.0](https://github.com/HDRUK/schemata-2/blob/master/hdr_schemata/models/HDRUK/2.1.0/schema.json) - [HDRUK 2.0.2](https://github.com/HDRUK/schemata-2/blob/master/hdr_schemata/models/HDRUK/2.0.2/schema.json) - **Other Schemas:** - [BioSchema](https://bioschemas.org/) - we can accept data conforming to this schema definition, but it is more limited - - [Gateway Data Model (GWDM 1.1)](https://hdruk.github.io/schemata-2/GWDM/1.1/) - a more general and flexible model. We use this schema internally to store any metadata that we accept from external sources. + - [Gateway Data Model (GWDM 2.0)](https://hdruk.github.io/schemata-2/GWDM/2.0/) - a more general and flexible model. We use this schema internally to store any metadata that we accept from external sources. We also support additional schemas, external to HDRUK, such as [BioSchema](https://bioschemas.org/). When you create your metadata and `POST` it to our APIs, the [translation service](https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/docs/) converts this metadata into the Gateway Data Model (GWDM) for internal storage. -- Example: `HDRUK 2.2.0` → `GWDM 1.1` +- Example: `HDRUK 3.0.0` → `GWDM 2.0` There are several schemas, each with possibly multiple versions, that we support. Metadata in each can be accepted as input, validated and translated into our GWDM. @@ -55,12 +56,14 @@ Schemas are published as [pydantic](https://docs.pydantic.dev/latest/) models th "2.1.0", "2.0.2", "2.2.1", - "2.2.0" + "2.2.0", + "3.0.0" ], "GWDM": [ "1.0", "1.1", - "1.2" + "1.2", + "2.0" ], "SchemaOrg": [ "default", @@ -78,7 +81,7 @@ Schemas are published as [pydantic](https://docs.pydantic.dev/latest/) models th Returns: ```bash - {"HDRUK":["2.1.2","2.1.3","2.1.0","2.0.2","2.2.1","2.2.0"],"GWDM":["1.0","1.1","1.2"],"SchemaOrg":["default","BioSchema","GoogleRecommended"]} + {"HDRUK":["2.1.2","2.1.3","2.1.0","2.0.2","2.2.1","2.2.0","3.0.0"],"GWDM":["1.0","1.1","1.2","2.0"],"SchemaOrg":["default","BioSchema","GoogleRecommended"]} ``` === "Web Browser" @@ -123,6 +126,12 @@ Our translation service will also allow you to see what available translations t "input_model": "HDRUK", "input_version": "2.0.2" }, + { + "output_model": "HDRUK", + "output_version": "3.0.0", + "input_model": "HDRUK", + "input_version": "2.2.1" + }, { "output_model": "HDRUK", "output_version": "2.1.2", @@ -213,6 +222,12 @@ Our translation service will also allow you to see what available translations t "input_model": "GWDM", "input_version": "1.2" }, + { + "output_model": "GWDM", + "output_version": "2.0", + "input_model": "GWDM", + "input_version": "1.2" + }, { "output_model": "GWDM", "output_version": "1.1", @@ -225,6 +240,12 @@ Our translation service will also allow you to see what available translations t "input_model": "HDRUK", "input_version": "2.2.1" }, + { + "output_model": "GWDM", + "output_version": "2.0", + "input_model": "HDRUK", + "input_version": "3.0.0" + }, { "output_model": "GWDM", "output_version": "1.2", @@ -266,6 +287,12 @@ Our translation service will also allow you to see what available translations t "output_version": "GoogleRecommended", "input_model": "GWDM", "input_version": "1.0" + }, + { + "output_model": "SchemaOrg", + "output_version": "GoogleRecommended", + "input_model": "GWDM", + "input_version": "2.0" } ] @@ -279,136 +306,154 @@ Our translation service will also allow you to see what available translations t Returns: ```bash - [{"output_model":"HDRUK","output_version":"2.1.2","input_model":"HDRUK","input_version":"datasetv2"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"HDRUK","input_version":"2.1.3"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"HDRUK","input_version":"2.0.2"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"GWDM","input_version":"1.0"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"GWDM","input_version":"1.1"},{"output_model":"HDRUK","output_version":"2.1.3","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"HDRUK","output_version":"2.2.1","input_model":"HDRUK","input_version":"2.2.0"},{"output_model":"HDRUK","output_version":"2.2.1","input_model":"GWDM","input_version":"1.2"},{"output_model":"HDRUK","output_version":"2.2.0","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"HDRUK","output_version":"2.2.0","input_model":"HDRUK","input_version":"2.2.1"},{"output_model":"HDRUK","output_version":"2.2.0","input_model":"GWDM","input_version":"1.1"},{"output_model":"GWDM","output_version":"1.0","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"GWDM","output_version":"1.0","input_model":"GWDM","input_version":"1.1"},{"output_model":"GWDM","output_version":"1.0","input_model":"SchemaOrg","input_version":"default"},{"output_model":"GWDM","output_version":"1.1","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"GWDM","output_version":"1.1","input_model":"HDRUK","input_version":"2.2.0"},{"output_model":"GWDM","output_version":"1.1","input_model":"GWDM","input_version":"1.0"},{"output_model":"GWDM","output_version":"1.1","input_model":"GWDM","input_version":"1.2"},{"output_model":"GWDM","output_version":"1.1","input_model":"SchemaOrg","input_version":"BioSchema"},{"output_model":"GWDM","output_version":"1.2","input_model":"HDRUK","input_version":"2.2.1"},{"output_model":"GWDM","output_version":"1.2","input_model":"GWDM","input_version":"1.1"},{"output_model":"SchemaOrg","output_version":"default","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"SchemaOrg","output_version":"default","input_model":"GWDM","input_version":"1.1"},{"output_model":"SchemaOrg","output_version":"BioSchema","input_model":"GWDM","input_version":"1.0"},{"output_model":"SchemaOrg","output_version":"BioSchema","input_model":"GWDM","input_version":"1.1"},{"output_model":"SchemaOrg","output_version":"GoogleRecommended","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"SchemaOrg","output_version":"GoogleRecommended","input_model":"GWDM","input_version":"1.0"}] + [{"output_model":"HDRUK","output_version":"2.1.2","input_model":"HDRUK","input_version":"datasetv2"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"HDRUK","input_version":"2.1.3"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"HDRUK","input_version":"2.0.2"},{"output_model":"HDRUK","output_version":"3.0.0","input_model":"HDRUK","input_version":"2.2.1"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"GWDM","input_version":"1.0"},{"output_model":"HDRUK","output_version":"2.1.2","input_model":"GWDM","input_version":"1.1"},{"output_model":"HDRUK","output_version":"2.1.3","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"HDRUK","output_version":"2.2.1","input_model":"HDRUK","input_version":"2.2.0"},{"output_model":"HDRUK","output_version":"2.2.1","input_model":"GWDM","input_version":"1.2"},{"output_model":"HDRUK","output_version":"2.2.0","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"HDRUK","output_version":"2.2.0","input_model":"HDRUK","input_version":"2.2.1"},{"output_model":"HDRUK","output_version":"2.2.0","input_model":"GWDM","input_version":"1.1"},{"output_model":"GWDM","output_version":"1.0","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"GWDM","output_version":"1.0","input_model":"GWDM","input_version":"1.1"},{"output_model":"GWDM","output_version":"1.0","input_model":"SchemaOrg","input_version":"default"},{"output_model":"GWDM","output_version":"1.1","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"GWDM","output_version":"1.1","input_model":"HDRUK","input_version":"2.2.0"},{"output_model":"GWDM","output_version":"1.1","input_model":"GWDM","input_version":"1.0"},{"output_model":"GWDM","output_version":"1.1","input_model":"GWDM","input_version":"1.2"},{"output_model":"GWDM","output_version":"2.0","input_model":"GWDM","input_version":"1.2"},{"output_model":"GWDM","output_version":"1.1","input_model":"SchemaOrg","input_version":"BioSchema"},{"output_model":"GWDM","output_version":"1.2","input_model":"HDRUK","input_version":"2.2.1"},{"output_model":"GWDM","output_version":"2.0","input_model":"HDRUK","input_version":"3.0.0"},{"output_model":"GWDM","output_version":"1.2","input_model":"GWDM","input_version":"1.1"},{"output_model":"SchemaOrg","output_version":"default","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"SchemaOrg","output_version":"default","input_model":"GWDM","input_version":"1.1"},{"output_model":"SchemaOrg","output_version":"BioSchema","input_model":"GWDM","input_version":"1.0"},{"output_model":"SchemaOrg","output_version":"BioSchema","input_model":"GWDM","input_version":"1.1"},{"output_model":"SchemaOrg","output_version":"GoogleRecommended","input_model":"HDRUK","input_version":"2.1.2"},{"output_model":"SchemaOrg","output_version":"GoogleRecommended","input_model":"GWDM","input_version":"1.0"},{"output_model":"SchemaOrg","output_version":"GoogleRecommended","input_model":"GWDM","input_version":"2.0"}] ``` === "Web Browser" [https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/list/templates](https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/list/templates) -## HDRUK 2.2.0 Template +## HDRUK 3.0.0 Template -You can fill out the following `JSON` template that matches our HDRUK `2.2.0` schema. +You can fill out the following `JSON` template that matches our HDRUK `3.0.0` schema. For each value in the `JSON` the template indicates the datatype that should/could be filled so that it passes the metadata validation step. ```json { - "identifier": Uuidv4[{'maxLength': 36, 'minLength': 36, 'pattern': '^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$', 'type': 'string'}] | Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null, - "version": Semver[{'pattern': '^([0-9]+)\\\\.([0-9]+)\\\\.([0-9]+)$', 'type': 'string'}], + "identifier": "Uuidv4[{'maxLength': 36, 'minLength': 36, 'pattern': '^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$', 'type': 'string'}] | Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null", + "version": "Semver[{'pattern': '^([0-9]+)\\\\.([0-9]+)\\\\.([0-9]+)$', 'type': 'string'}]", "revisions": { - "version": Semver[{'pattern': '^([0-9]+)\\\\.([0-9]+)\\\\.([0-9]+)$', 'type': 'string'}], - "url": Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null + "version": "Semver[{'pattern': '^([0-9]+)\\\\.([0-9]+)\\\\.([0-9]+)$', 'type': 'string'}]", + "url": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null" }, - "issued": datetime, - "modified": datetime, + "issued": "datetime", + "modified": "datetime", "summary": { - "title": OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}], - "abstract": AbstractText[{'anyOf': [{'maxLength': 500, 'minLength': 5, 'type': 'string'}, {'type': 'null'}]}] | null, - "publisher": { - "identifier": Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null, - "name": OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}], - "logo": Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null, - "description": Description[{'anyOf': [{'maxLength': 10000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "contactPoint": EmailAddress[{'anyOf': [{'format': 'email', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.EmailAddress.EmailAddress]] | null, - "memberOf": MemberOf['HUB','ALLIANCE','OTHER','NCS'] | null + "title": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}]", + "abstract": "AbstractText[{'anyOf': [{'maxLength': 500, 'minLength': 5, 'type': 'string'}, {'type': 'null'}]}]", + "dataProvider": { + "identifier": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null", + "name": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}]", + "logo": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null", + "description": "Description[{'anyOf': [{'maxLength': 10000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null", + "contactPoint": "EmailAddress[{'anyOf': [{'format': 'email', 'type': 'string'}, {'type': 'null'}]}] | List", + "memberOf": "MemberOf['HUB','ALLIANCE','OTHER','NCS'] | null" }, - "contactPoint": EmailAddress[{'anyOf': [{'format': 'email', 'type': 'string'}, {'type': 'null'}]}] | null, - "keywords": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.OneHundredFiftyCharacters.OneHundredFiftyCharacters] | null, - "alternateIdentifiers": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.ShortDescription.ShortDescription]] | null, - "doiName": Doi[{'anyOf': [{'pattern': '^10.\\\\d{4,9}/[-._;()/:a-zA-Z0-9]+$', 'type': 'string'}, {'type': 'null'}]}] | null, - "datasetType": DatasetType[{'anyOf': [{'maxLength': 100, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "datasetSubType": DatasetType[{'anyOf': [{'maxLength': 100, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "populationSize": int | null + "populationSize": "int", + "keywords": "CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | List | null", + "doiName": "Doi[{'anyOf': [{'pattern': '^10.\\\\d{4,9}/[-._;()/:a-zA-Z0-9]+$', 'type': 'string'}, {'type': 'null'}]}] | null", + "contactPoint": "EmailAddress[{'anyOf': [{'format': 'email', 'type': 'string'}, {'type': 'null'}]}]", + "alternateIdentifiers": "CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | List | null" }, "documentation": { - "description": Description[{'anyOf': [{'maxLength': 10000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "associatedMedia": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.Url.Url]] | null, - "isPartOf": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Union[hdr_schemata.definitions.HDRUK.Url.Url, NoneType, hdr_schemata.definitions.HDRUK.OneHundredFiftyCharacters.OneHundredFiftyCharacters, hdr_schemata.definitions.HDRUK.IsPartOfEnum.IsPartOfEnum]] | null + "description": "Description[{'anyOf': [{'maxLength': 10000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}]", + "associatedMedia": "CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | List | null", + "inPipeline": "Pipeline['Available','Not available'] | null" }, "coverage": { - "spatial": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | null, - "pathway": LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "followup": Followup['0 - 6 MONTHS','6 - 12 MONTHS','1 - 10 YEARS','> 10 YEARS','UNKNOWN','CONTINUOUS','OTHER',null] | null, - "typicalAgeRange": AgeRange[{'anyOf': [{'pattern': 'Not Known|(150|1[0-4][0-9]|[0-9]|[1-8][0-9]|9[0-9])-(150|1[0-4][0-9]|[0-9]|[1-8][0-9]|9[0-9])', 'type': 'string'}, {'type': 'null'}]}] | null, - "gender": GenderType['Male','Female','Other'], - "biologicalsamples": BiologicalSampleType['Blood','Other','Urine','Saliva'], - "psychological": PsychologicalType['Cognitive Function','Mental Health'], - "physical": PhysicalType['Respiratory','Vision','Hearing','Musculoskeletal','Cardiovascular','Reproductive'], - "anthropometric": AnthropometricType['Blood Pressure','Hip Circumference','Height','Waist Circumference','Weight'], - "lifestyle": LifestylesType['Smoking','Dietary Habits','Physical Activity','Alcohol'], - "socioeconomic": SocioEconomicType['Finances','Family Circumstances','Housing','Education','Marital Status','Occupation','Ethnic Group','Social Support'] + "spatial": "CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | List", + "typicalAgeRangeMin": "int | null", + "typicalAgeRangeMax": "int | null", + "datasetCompleteness": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null", + "materialType": "MaterialTypeCategoriesV2['None/not available','Bone marrow','Cancer cell lines','CDNA/MRNA','Core biopsy','DNA','Entire body organ','Faeces','Immortalized cell lines','Isolated pathogen','MicroRNA','Peripheral blood cells','Plasma','PM Tissue','Primary cells','RNA','Saliva','Serum','Swabs','Tissue','Urine','Whole blood','Availability to be confirmed','Other']", + "followup": "Followup['0 - 6 MONTHS','6 - 12 MONTHS','1 - 10 YEARS','> 10 YEARS','UNKNOWN','CONTINUOUS','OTHER',null] | null", + "pathway": "Description[{'anyOf': [{'maxLength': 10000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null", + "gender": "GenderType['Male','Female','Other']" }, "provenance": { "origin": { - "purpose": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.Purpose.Purpose] | null, - "source": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.Source.Source] | null, - "collectionSituation": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.Setting.Setting] | null + "purpose": "PurposeV2['Research cohort','Study','Disease registry','Trial','Care','Audit','Administrative','Financial','Statuatory','Other',null]", + "datasetType": "DatasetTypeV2['Health and disease','Treatments/Interventions','Measurements/Tests','Imaging types','Imaging area of the body','Omics','Socioeconomic','Lifestyle','Registry','Environment and energy','Information and communication','Politics']", + "datasetSubType": "DatasetSubType['Mental health','Cardiovascular','Cancer','Rare diseases','Metabolic and Endocrine','Neurological','Reproductive','Maternity and neonatology','Respiratory','Immunity','Musculoskeletal','Vision','Renal and urogenital','Oral and Gastrointestinal','Cognitive Function','Hearing','Others','Vaccines','Preventive','Therapeutic','Laboratory','Other diagnostics','CT','MRI','PET','X-ray','Ultrasound','Pathology','Head','Chest','Arm','Abdomen','Leg','Proteomics','Transcriptomics','Epigenomics','Metabolomics','Multiomics','Metagenomics','Genomics','Education','Crime and Justice','Ethnicity','Housing ','Labour','Ageing ','Economics','Marital status','Social support','Deprivation','Religion','Occupation','Finances','Family circumstance','Smoking','Physical Activity','Dietary habits','Alcohol','Disease Registry (research)','National Disease Registries and Audits','Births and Deaths','Not applicable'] | null", + "source": "SourceV2['EPR','Electronic survey','LIMS','Paper-based','Free text NLP','Machine generated','Other']", + "collectionSource": "SettingV2['Cohort, study, trial','Clinic','Primary care - Referrals','Primary care - Clinic','Primary care - Out of hours','Secondary care - Accident and Emergency','Secondary care - Outpatients','Secondary care - In-patients','Secondary care - Ambulance','Secondary care - ICU','Prescribing - Community pharmacy','Prescribing - Hospital','Patient report outcome','Wearables','Local authority','National government','Community','Services','Home','Private','Social care - Health care at home','Social care - Other social data','Census','Other',null]", + "imageContrast": "Ternary['Yes','No','Not stated'] | null" }, "temporal": { - "distributionReleaseDate": date | datetime | null, - "startDate": date | datetime | null, - "endDate": date | datetime | EndDateEnum['CONTINUOUS',null] | null, - "timeLag": TimeLag['LESS 1 WEEK','1-2 WEEKS','2-4 WEEKS','1-2 MONTHS','2-6 MONTHS','MORE 6 MONTHS','VARIABLE','NO TIMELAG','NOT APPLICABLE','OTHER',null], - "publishingFrequency": Periodicity['STATIC','IRREGULAR','CONTINUOUS','BIENNIAL','ANNUAL','BIANNUAL','QUARTERLY','BIMONTHLY','MONTHLY','BIWEEKLY','WEEKLY','SEMIWEEKLY','DAILY','OTHER',null] + "publishingFrequency": "PeriodicityV2['Static','Irregular','Continuous','Biennial','Annual','Biannual','Quarterly','Bimonthly','Monthly','Biweekly','Weekly','Twice a week','Daily','Other',null]", + "distributionReleaseDate": "date | datetime | null", + "startDate": "date | datetime", + "endDate": "date | datetime | EndDateEnum['CONTINUOUS',null] | null", + "timeLag": "TimeLagV2['Less than 1 week','1-2 weeks','2-4 weeks','1-2 months','2-6 months','6 months plus','Variable','Not applicable','Other']" } }, "accessibility": { "usage": { - "dataUseLimitation": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.DataUseLimitation.DataUseLimitation] | null, - "dataUseRequirements": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.DataUseRequirements.DataUseRequirements] | null, - "resourceCreator": ShortDescription[{'anyOf': [{'maxLength': 1000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.ShortDescription.ShortDescription]] | null, - "investigations": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.Url.Url]] | null, - "isReferencedBy": Doi[{'anyOf': [{'pattern': '^10.\\\\d{4,9}/[-._;()/:a-zA-Z0-9]+$', 'type': 'string'}, {'type': 'null'}]}] | str | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.Doi.Doi]] | null + "dataUseLimitation": "DataUseLimitationV2['General research use','Genetic studies only','No general methods research','No restriction','Research-specific restrictions','Research use only','No linkage']", + "dataUseRequirements": "DataUseRequirementsV2['Collaboration required','Ethics approval required','Geographical restrictions','Institution-specific restrictions','Not for profit use','Project-specific restrictions','Publication moratorium','Publication required','Return to database or resource','Time limit on use','User-specific restriction']", + "resourceCreator": "ShortDescription[{'anyOf': [{'maxLength': 1000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | List | null" }, "access": { - "accessRights": LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "accessService": LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "accessRequestCost": LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.Url.Url]] | null, - "deliveryLeadTime": DeliveryLeadTime['LESS 1 WEEK','1-2 WEEKS','2-4 WEEKS','1-2 MONTHS','2-6 MONTHS','MORE 6 MONTHS','VARIABLE','NOT APPLICABLE','OTHER',null] | null, - "jurisdiction": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.Isocountrycode.Isocountrycode] | null, - "dataController": LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null, - "dataProcessor": LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null + "accessRights": "LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}]", + "accessServiceCategory": "AccessService['TRE/SDE','Direct access','Open access','Varies based on project'] | null", + "accessMode": "AccessMode['Join research consortium','New project'] | null", + "accessService": "LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null", + "accessRequestCost": "LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null", + "deliveryLeadTime": "DeliveryLeadTimeV2['Less than 1 week','1-2 weeks','2-4 weeks','1-2 months','2-6 months','More than 6 months','Variable','Not applicable','Other'] | null", + "jurisdiction": "CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | List | null", + "dataController": "LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null", + "dataProcessor": "LongDescription[{'anyOf': [{'maxLength': 50000, 'minLength': 2, 'type': 'string'}, {'type': 'null'}]}] | null" }, "formatAndStandards": { - "vocabularyEncodingScheme": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.ControlledVocabulary.ControlledVocabulary] | null, - "conformsTo": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.StandardisedDataModels.StandardisedDataModels] | null, - "language": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.Language.Language] | null, - "format": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[hdr_schemata.definitions.HDRUK.Format.Format] | null + "vocabularyEncodingScheme": "ControlledVocabulary[{'$defs': {'ControlledVocabularyEnum': {'enum': ['LOCAL', 'OPCS4', 'READ', 'SNOMED CT', 'SNOMED RT', 'DM PLUS D', 'DM+D', 'NHS NATIONAL CODES', 'NHS SCOTLAND NATIONAL CODES', 'NHS WALES NATIONAL CODES', 'ODS', 'LOINC', 'ICD10', 'ICD10CM', 'ICD10PCS', 'ICD9CM', 'ICD9', 'ICDO3', 'AMT', 'APC', 'ATC', 'CIEL', 'HPO', 'CPT4', 'DPD', 'DRG', 'HEMONC', 'JMDC', 'KCD7', 'MULTUM', 'NAACCR', 'NDC', 'NDFRT', 'OXMIS', 'RXNORM', 'RXNORM EXTENSION', 'SPL', 'OTHER'], 'title': 'ControlledVocabularyEnum', 'type': 'string'}}, 'anyOf': [{'$ref': '#/$defs/ControlledVocabularyEnum'}, {'type': 'null'}], 'default': null}]", + "conformsTo": "StandardisedDataModels[{'$defs': {'StandardisedDataModelsEnum': {'enum': ['HL7 FHIR', 'HL7 V2', 'HL7 CDA', 'HL7 CCOW', 'LOINC', 'DICOM', 'I2B2', 'IHE', 'OMOP', 'OPENEHR', 'SENTINEL', 'PCORNET', 'CDISC', 'NHS DATA DICTIONARY', 'NHS SCOTLAND DATA DICTIONARY', 'NHS WALES DATA DICTIONARY', 'LOCAL', 'OTHER'], 'title': 'StandardisedDataModelsEnum', 'type': 'string'}}, 'anyOf': [{'$ref': '#/$defs/StandardisedDataModelsEnum'}, {'type': 'null'}], 'default': null}]", + "language": "CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | List | null", + "format": "CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | List | null" } }, "enrichmentAndLinkage": { - "qualifiedRelation": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Union[hdr_schemata.definitions.HDRUK.Url.Url, NoneType, hdr_schemata.definitions.HDRUK.OneHundredFiftyCharacters.OneHundredFiftyCharacters]] | null, - "derivation": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.AbstractText.AbstractText]] | null, - "tools": CommaSeparatedValues[{'anyOf': [{'pattern': '([^,]+)', 'type': 'string'}, {'type': 'null'}]}] | typing.List[typing.Optional[hdr_schemata.definitions.HDRUK.Url.Url]] | null + "derivedFrom": { + "pid": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "title": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "url": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null" + }, + "isPartOf": { + "pid": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "title": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "url": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null" + }, + "linkableDatasets": { + "pid": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "title": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "url": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null" + }, + "similarToDatasets": { + "pid": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "title": "OneHundredFiftyCharacters[{'maxLength': 150, 'minLength': 2, 'type': 'string'}] | null", + "url": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}] | null" + }, + "investigations": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}]", + "tools": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}]", + "publicationAboutDataset": "Doi[{'anyOf': [{'pattern': '^10.\\\\d{4,9}/[-._;()/:a-zA-Z0-9]+$', 'type': 'string'}, {'type': 'null'}]}]", + "publicationUsingDataset": "Doi[{'anyOf': [{'pattern': '^10.\\\\d{4,9}/[-._;()/:a-zA-Z0-9]+$', 'type': 'string'}, {'type': 'null'}]}]" }, - "observations": [{ - "observedNode": StatisticalPopulationConstrained['PERSONS','EVENTS','FINDINGS'], - "measuredValue": int, - "disambiguatingDescription": AbstractText[{'anyOf': [{'maxLength': 500, 'minLength': 5, 'type': 'string'}, {'type': 'null'}]}] | null, - "observationDate": date | datetime, - "measuredProperty": MeasuredProperty[{}] - }], - "structuralMetadata": [{ - "name": str | null, - "description": str | null, - "elements": { - "name": Name[{}], - "dataType": str, - "description": str | null, - "sensitive": bool - } - }], - "tissuesSampleCollection": [{ - "dataCategories": TissueDataCategoriesEnum['Biological samples','Survey data','Imaging data','Medical records','National registries','Genealogical records','Physiological/Biochemical measurements','Other'], - "materialType": MaterialTypeCategories['Blood','DNA','Faeces','Immortalized Cell Lines','Isolated Pathogen','Other','Plasma','RNA','Saliva','Serum','Tissue (Frozen)','Tissue (FFPE)','Urine'], - "tissueSampleMetadata": { - "creationDate": date | datetime | null, - "AnatomicalSiteOntologyCode": ICD_0_3[{'anyOf': [{'pattern': '^[C\\\\d]{3}\\\\.\\\\d{4}\\\\/\\\\d{1,4}$', 'type': 'string'}, {'type': 'null'}]}] | null + "observations": { + "observedNode": "StatisticalPopulationConstrainedV2['Person','Event','Findings','Number of scans per modality']", + "measuredValue": "int", + "disambiguatingDescription": "AbstractText[{'anyOf': [{'maxLength': 500, 'minLength': 5, 'type': 'string'}, {'type': 'null'}]}] | null", + "observationDate": "date | datetime", + "measuredProperty": "MeasuredProperty[{}]" + }, + "structuralMetadata": { + "tables": { + "name": "str | null", + "description": "str | null", + "columns": { + "name": "Name[{}]", + "dataType": "str", + "description": "str | null", + "sensitive": "bool", + "values": { + "name": "Name[{}]", + "description": "str | null", + "frequency": "int | null" + } + } }, - "collectionType": TissueCollectionTypeEnum['Case-control','Cohort','Cross-sectional','Longitudinal','Twin-study','Quality control','Population-based','Disease specific','Birth cohort','Other'] | null - }] + "syntheticDataWebLink": "Url[{'anyOf': [{'format': 'uri', 'minLength': 1, 'type': 'string'}, {'type': 'null'}]}]" + } } diff --git a/docs/example-metadata.md b/docs/example-metadata.md index 078b7bf..7ffbf07 100644 --- a/docs/example-metadata.md +++ b/docs/example-metadata.md @@ -2764,6 +2764,172 @@ The change log for this schema can be found at: } ``` +### HDR v3.0.0 + + ```json + { + "identifier": "https://web.www.healthdatagateway.org/19525c5f-92ee-41b6-bb79-673624b27bdd", + "version": "3.0.0", + "issued": "2021-05-10T00:00:00.000Z", + "modified": "2021-05-10T00:00:00.000Z", + "revisions": [], + "summary": { + "title": "Improving Access to Psychological Therapies Data Set", + "abstract": "Patient-level data set that captures information about people in contact with services commissioned as part of the adult Improving Access to Psychological Services (IAPT) programme.", + "contactPoint": "enquiries@nhsdigital.nhs.uk", + "keywords": " Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS", + "alternateIdentifiers": null, + "doiName": null, + "populationSize": -1, + "dataProvider": { + "identifier": "https://web.www.healthdatagateway.org/5f86cd34980f41c6f02261f4", + "name": "NHS DIGITAL", + "logo": null, + "description": null, + "contactPoint": null, + "memberOf": "ALLIANCE" + } + }, + "documentation": { + "description": "Collecting information about people in contact with adult psychological therapy services in England. The IAPT data set was developed with the IAPT programme as a patient level, output based, secondary uses data set which aims to deliver robust, comprehensive, nationally consistent and comparable information for patients accessing NHS-funded IAPT services in England. This national data set has been collected since April 2012 and is a mandatory submission for all NHS funded care, including care delivered by independent sector healthcare providers. Data collection on patients with depression and anxiety disorders that are offered psychological therapies, so that we can improve the delivery of care for these conditions.\n\nProviders of NHS-funded IAPT services are required to submit data to NHS Digital on a monthly basis.\n\nAs a secondary uses data set the IAPT data set re-uses clinical and operational data for purposes other than direct patient care. It defines the data items, definitions and associated value sets extracted or derived from local information systems and sent to NHS Digital for analysis purposes. Timescales for dissemination can be found under 'Our Service Levels' at the following link: https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-process", + "associatedMedia": null, + "inPipeline": null + }, + "coverage": { + "spatial": "United Kingdom,England", + "followup": "0 - 6 MONTHS", + "pathway": null, + "gender": null, + "typicalAgeRangeMin": 18, + "typicalAgeRangeMax": 150, + "datasetCompleteness": null, + "materialType": [ + "None/not available" + ] + }, + "provenance": { + "origin": { + "purpose": [ + "Other", + "Administrative" + ], + "source": [ + "EPR" + ], + "datasetType": "Health and disease", + "datasetSubType": "Not applicable", + "collectionSource": null, + "imageContrast": null + }, + "temporal": { + "distributionReleaseDate": null, + "startDate": "2012-04-01", + "endDate": null, + "timeLag": "2-6 months", + "publishingFrequency": "Monthly" + } + }, + "accessibility": { + "usage": { + "dataUseLimitation": [ + "No restriction" + ], + "dataUseRequirements": null, + "resourceCreator": null + }, + "access": { + "accessRights": "https://digital.nhs.uk/services/data-access-request-service-dars", + "accessService": "Once your DARS application has been approved, data will be made available either by secure file transfer or through the Data Access Environment (DAE). BL\n\nSecure file transfer: https://digital.nhs.uk/services/transfer-data-securely\n\nDAE: https://digital.nhs.uk/services/data-access-environment-dae", + "accessRequestCost": "https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-charges", + "deliveryLeadTime": null, + "jurisdiction": "GB-ENG", + "dataProcessor": null, + "dataController": "NHS DIGITAL", + "accessServiceCategory": null, + "accessMode": "New project" + }, + "formatAndStandards": { + "vocabularyEncodingScheme": [ + "ODS", + "SNOMED CT", + "NHS NATIONAL CODES", + "ICD10" + ], + "conformsTo": [ + "NHS DATA DICTIONARY" + ], + "language": "en", + "format": "CSV" + } + }, + "enrichmentAndLinkage": { + "tools": null, + "derivedFrom": null, + "isPartOf": null, + "linkableDatasets": null, + "similarToDatasets": null, + "publicationAboutDataset": null, + "investigations": null, + "publicationUsingDataset": null + }, + "observations": [ + { + "observedNode": "Event", + "measuredValue": 114125, + "measuredProperty": "COUNT", + "observationDate": "2021-02-25", + "disambiguatingDescription": "In December 2020 there were 114,125 referrals made into IAPT services in England." + }, + { + "observedNode": "Event", + "measuredValue": 81578, + "measuredProperty": "COUNT", + "observationDate": "2021-02-25", + "disambiguatingDescription": "In December 2020, 81,578 referrals into IAPT services in England started a course of treatment." + }, + { + "observedNode": "Event", + "measuredValue": 48331, + "measuredProperty": "COUNT", + "observationDate": "2021-02-25", + "disambiguatingDescription": "In December 2020, 48,331 referrals into IAPT services in England completed a course of treatment." + } + ], + "structuralMetadata": { + "tables": [ + { + "name": "IAPT.iapt.Rep_Referral", + "description": "IAPT.iapt.Rep_Referral", + "columns": [ + { + "name": "Count of number of Non-guided Self Help (Computer) sessions (derived)", + "description": "Count of number of Non-guided Self Help (Computer) sessions (derived)", + "dataType": "Number", + "sensitive": false, + "values": null + } + ] + }, + { + "name": "IAPT.iapt.Rep_Referral", + "description": "IAPT.iapt.Rep_Referral", + "columns": [ + { + "name": "Pseudonymised Service Request Identifier", + "description": "A request for the provision of care services to a PATIENT.", + "dataType": "String", + "sensitive": false, + "values": null + } + ] + } + ], + "syntheticDataWebLink": null + } + } + + ``` + ### BioSchema This example payload (`{"metadata":}`) will validate (return status `200`) against TRASER (translation service) endpoint: diff --git a/docs/metadata-validation.md b/docs/metadata-validation.md index c293f50..ecf5739 100644 --- a/docs/metadata-validation.md +++ b/docs/metadata-validation.md @@ -1,8 +1,8 @@ ## Schema Validation -Once you have created your metadata, for example for the HDRUK schema `2.1.2`, you can use [various routes](https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/docs/) of the translation service to check to see if it validates against this schema. +Once you have created your metadata, for example for the HDRUK schema `3.0.0`, you can use [various routes](https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/docs/) of the translation service to check to see if it validates against this schema. -Using the example file found at [https://github.com/HDRUK/gateway-2-integrations-testing/blob/master/example-hdruk212.json](https://github.com/HDRUK/gateway-2-integrations-testing/blob/master/example-hdruk212.json): +Using the example file found at [https://github.com/HDRUK/gateway-2-integrations-testing/blob/master/example-hdruk300.json](https://github.com/HDRUK/gateway-2-integrations-testing/blob/master/example-hdruk300.json): === "python" @@ -12,7 +12,7 @@ Using the example file found at [https://github.com/HDRUK/gateway-2-integrations import json import requests - metadata = json.load(open("example-hdruk212.json")) + metadata = json.load(open("example-hdruk300.json")) headers = { "Content-Type": "application/json", @@ -30,7 +30,7 @@ Using the example file found at [https://github.com/HDRUK/gateway-2-integrations === "CURL" ```bash - metadata='{"metadata": '$(cat example-hdruk212.json)'}' + metadata='{"metadata": '$(cat example-hdruk300.json)'}' curl --location 'https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/find?with_errors=1'\ --header 'Content-Type: application/json' --data \ --data "$metadata" @@ -41,218 +41,218 @@ Responds with: ``` [ { - "name": "HDRUK", - "version": "2.1.2", - "matches": true, - "errors": null + "name": "HDRUK", + "version": "2.1.2", + "matches": false, + "errors": [ + { + "instancePath": "/summary", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "publisher" + }, + "message": "must have required property 'publisher'" + } + ] }, { - "name": "HDRUK", - "version": "2.1.3", - "matches": false, - "errors": [ - { - "instancePath": "/provenance/temporal", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "publishingFrequency" - }, - "message": "must have required property 'publishingFrequency'" - }, - { - "instancePath": "/provenance/temporal", - "schemaPath": "#/properties/temporal/anyOf/1/type", - "keyword": "type", - "params": { - "type": "null" - }, - "message": "must be null" + "name": "HDRUK", + "version": "2.1.3", + "matches": false, + "errors": [ + { + "instancePath": "/summary", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "publisher" }, - { - "instancePath": "/provenance/temporal", - "schemaPath": "#/properties/temporal/anyOf", - "keyword": "anyOf", - "params": {}, - "message": "must match a schema in anyOf" + "message": "must have required property 'publisher'" + } + ] + }, + { + "name": "HDRUK", + "version": "2.1.0", + "matches": false, + "errors": [ + { + "instancePath": "/summary", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "publisher" }, - { - "instancePath": "/provenance", - "schemaPath": "#/properties/provenance/anyOf/1/type", - "keyword": "type", - "params": { - "type": "null" - }, - "message": "must be null" + "message": "must have required property 'publisher'" + } + ] + }, + { + "name": "HDRUK", + "version": "2.0.2", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/additionalProperties", + "keyword": "additionalProperties", + "params": { + "additionalProperty": "structuralMetadata" }, - { - "instancePath": "/provenance", - "schemaPath": "#/properties/provenance/anyOf", - "keyword": "anyOf", - "params": {}, - "message": "must match a schema in anyOf" - } - ] + "message": "must NOT have additional properties" + } + ] }, { - "name": "HDRUK", - "version": "2.1.0", - "matches": false, - "errors": [ - { - "instancePath": "/summary/publisher/logo", - "schemaPath": "#/definitions/url/format", - "keyword": "format", - "params": { - "format": "uri" - }, - "message": "must match format \"uri\"" - } - ] + "name": "HDRUK", + "version": "2.2.1", + "matches": false, + "errors": [ + { + "instancePath": "/summary", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "publisher" + }, + "message": "must have required property 'publisher'" + } + ] }, { - "name": "HDRUK", - "version": "2.0.2", - "matches": false, - "errors": [ - { - "instancePath": "", - "schemaPath": "#/additionalProperties", - "keyword": "additionalProperties", - "params": { - "additionalProperty": "structuralMetadata" - }, - "message": "must NOT have additional properties" - } - ] + "name": "HDRUK", + "version": "2.2.0", + "matches": false, + "errors": [ + { + "instancePath": "/summary", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "publisher" + }, + "message": "must have required property 'publisher'" + } + ] }, { - "name": "HDRUK", - "version": "2.2.1", - "matches": false, - "errors": [ - { - "instancePath": "/summary", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "datasetType" - }, - "message": "must have required property 'datasetType'" - } - ] + "name": "HDRUK", + "version": "3.0.0", + "matches": true, + "errors": null }, { - "name": "HDRUK", - "version": "2.2.0", - "matches": false, - "errors": [ - { - "instancePath": "/summary", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "datasetType" - }, - "message": "must have required property 'datasetType'" - } - ] + "name": "GWDM", + "version": "1.0", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "required" + }, + "message": "must have required property 'required'" + } + ] }, { - "name": "GWDM", - "version": "1.0", - "matches": false, - "errors": [ - { - "instancePath": "", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "required" - }, - "message": "must have required property 'required'" - } - ] + "name": "GWDM", + "version": "1.1", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "required" + }, + "message": "must have required property 'required'" + } + ] }, { - "name": "GWDM", - "version": "1.1", - "matches": false, - "errors": [ - { - "instancePath": "", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "required" - }, - "message": "must have required property 'required'" - } - ] + "name": "GWDM", + "version": "1.2", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "required" + }, + "message": "must have required property 'required'" + } + ] }, { - "name": "GWDM", - "version": "1.2", - "matches": false, - "errors": [ - { - "instancePath": "", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "required" - }, - "message": "must have required property 'required'" - } - ] + "name": "GWDM", + "version": "2.0", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "required" + }, + "message": "must have required property 'required'" + } + ] }, { - "name": "SchemaOrg", - "version": "default", - "matches": false, - "errors": [ - { - "instancePath": "", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "name" - }, - "message": "must have required property 'name'" - } - ] + "name": "SchemaOrg", + "version": "default", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "name" + }, + "message": "must have required property 'name'" + } + ] }, { - "name": "SchemaOrg", - "version": "BioSchema", - "matches": false, - "errors": [ - { - "instancePath": "", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "name" - }, - "message": "must have required property 'name'" - } - ] + "name": "SchemaOrg", + "version": "BioSchema", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "name" + }, + "message": "must have required property 'name'" + } + ] }, { - "name": "SchemaOrg", - "version": "GoogleRecommended", - "matches": false, - "errors": [ - { - "instancePath": "", - "schemaPath": "#/required", - "keyword": "required", - "params": { - "missingProperty": "name" - }, - "message": "must have required property 'name'" - } - ] + "name": "SchemaOrg", + "version": "GoogleRecommended", + "matches": false, + "errors": [ + { + "instancePath": "", + "schemaPath": "#/required", + "keyword": "required", + "params": { + "missingProperty": "name" + }, + "message": "must have required property 'name'" + } + ] } ] ``` @@ -266,7 +266,7 @@ You can use the route `validate` instead of `find` to validate a payload against import json import requests - metadata = json.load(open("example-hdruk212.json")) + metadata = json.load(open("example-hdruk300.json")) headers = { "Content-Type": "application/json", @@ -275,7 +275,7 @@ You can use the route `validate` instead of `find` to validate a payload against traser_uri = "https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app" response = requests.post( - f"{traser_uri}/validate?input_schema=HDRUK&input_version=2.1.2", + f"{traser_uri}/validate?input_schema=HDRUK&input_version=3.0.0", headers=headers, json={"metadata":metadata} ) @@ -321,8 +321,8 @@ You can use the route `validate` instead of `find` to validate a payload against === "CURL" ``` - metadata='{"metadata": '$(cat example-hdruk212.json)'}' - curl --location 'https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/validate?input_schema=HDRUK&input_version=2.1.2' \ + metadata='{"metadata": '$(cat example-hdruk300.json)'}' + curl --location 'https://hdr-gateway-traser-dev-qmnkcg5qjq-ew.a.run.app/validate?input_schema=HDRUK&input_version=3.0.0' \ --header 'Content-Type: application/json' \ --data "$metadata" ``` diff --git a/docs/retrieving-a-dataset.md b/docs/retrieving-a-dataset.md index 01c6fa4..0ff9b87 100644 --- a/docs/retrieving-a-dataset.md +++ b/docs/retrieving-a-dataset.md @@ -122,13 +122,13 @@ Sample output: ``` -#### HDRUK 2.2.0 (public schema) +#### HDRUK 3.0.0 (public schema) -Using `/api/v1/integrations/datasets/?schema_model=HDRUK&schema_version=2.2.0` you can get back your metadata conforming to our public schema from our Gateway Data Model (GWDM): +Using `/api/v1/integrations/datasets/?schema_model=HDRUK&schema_version=3.0.0` you can get back your metadata conforming to our public schema from our Gateway Data Model (GWDM): ```json { - "message": "success, translated to model=HDRUK version=2.2.0", + "message": "success, translated to model=HDRUK version=3.0.0", "data": { "identifier": "96bea284-dd48-4617-84e5-8b8f888b2fb3", "issued": "2021-05-10T00:00:00.000Z", diff --git a/example-gwdm12.json b/example-gwdm20.json similarity index 90% rename from example-gwdm12.json rename to example-gwdm20.json index d5262dc..009dee4 100644 --- a/example-gwdm12.json +++ b/example-gwdm20.json @@ -40,26 +40,22 @@ "rorId": null }, "populationSize": 907515, - "datasetSubType": null + "datasetSubType": null, + "inPipeline": null }, "coverage": { "pathway": "NOT APPLICABLE", "spatial": "https://www.geonames.org/countries/GB/united-kingdom.html", "followup": "UNKNOWN", "typicalAgeRange": "0-0", - "biologicalsamples": null, - "gender": null, - "psychological": null, - "physical": null, - "anthropometric": null, - "lifestyle": null, - "socioeconomic": null + "datasetCompleteness": null }, "provenance": { "origin": { "purpose": "OTHER", "source": "MACHINE GENERATED", - "collectionSituation": "OTHER" + "collectionSituation": "OTHER", + "imageContrast": null }, "temporal": { "endDate": "2022-04-30", @@ -78,7 +74,8 @@ "accessRights": "https://raw.githubusercontent.com/HDRUK/papers/master/LICENSE", "accessService": "https://github.com/HDRUK/papers", "accessRequestCost": "Free", - "accessServiceCategory": null + "accessServiceCategory": null, + "accessMode": "New project" }, "usage": { "dataUseLimitation": "GENERAL RESEARCH USE", @@ -109,18 +106,20 @@ }, "investigations": "https://github.com/HDRUK/papers", "associatedMedia": null, - "syntheticDataWebLink": null + "syntheticDataWebLink": null, + "publicationAboutDataset": null, + "publicationUsingDataset": null }, "observations": [ { - "observedNode": "PERSONS", + "observedNode": "Person", "measuredValue": 907515, "disambiguatingDescription": "Number of CPRD GOLD patients from June 2020 build with a cancer identifier in set 19", "observationDate": "2020-06-26", "measuredProperty": "Count" }, { - "observedNode": "FINDINGS", + "observedNode": "Findings", "measuredValue": 907515, "disambiguatingDescription": "Number of CPRD GOLD patients from June 2020 build with a cancer identifier in set 19", "observationDate": "2020-06-26", @@ -141,6 +140,5 @@ } ] } - ], - "tissuesSampleCollection": null + ] } \ No newline at end of file diff --git a/example-hdruk212.json b/example-hdruk300.json similarity index 59% rename from example-hdruk212.json rename to example-hdruk300.json index 209d686..725a9ff 100644 --- a/example-hdruk212.json +++ b/example-hdruk300.json @@ -7,52 +7,66 @@ "summary": { "title": "Improving Access to Psychological Therapies Data Set", "abstract": "Patient-level data set that captures information about people in contact with services commissioned as part of the adult Improving Access to Psychological Services (IAPT) programme.", - "publisher": { + "contactPoint": "enquiries@nhsdigital.nhs.uk", + "keywords": " Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS", + "alternateIdentifiers": null, + "doiName": null, + "populationSize": -1, + "dataProvider": { "identifier": "https://web.www.healthdatagateway.org/5f86cd34980f41c6f02261f4", "name": "NHS DIGITAL", - "logo": "", - "description": "", + "logo": null, + "description": null, "contactPoint": null, "memberOf": "ALLIANCE" - }, - "contactPoint": "enquiries@nhsdigital.nhs.uk", - "keywords": " Depression,Anxiety,CBT,Counselling,Talking theraphy,Cognitive,National Core Study,NCS", - "alternateIdentifiers": null, - "doiName": "" + } }, "documentation": { "description": "Collecting information about people in contact with adult psychological therapy services in England. The IAPT data set was developed with the IAPT programme as a patient level, output based, secondary uses data set which aims to deliver robust, comprehensive, nationally consistent and comparable information for patients accessing NHS-funded IAPT services in England. This national data set has been collected since April 2012 and is a mandatory submission for all NHS funded care, including care delivered by independent sector healthcare providers. Data collection on patients with depression and anxiety disorders that are offered psychological therapies, so that we can improve the delivery of care for these conditions.\n\nProviders of NHS-funded IAPT services are required to submit data to NHS Digital on a monthly basis.\n\nAs a secondary uses data set the IAPT data set re-uses clinical and operational data for purposes other than direct patient care. It defines the data items, definitions and associated value sets extracted or derived from local information systems and sent to NHS Digital for analysis purposes. Timescales for dissemination can be found under 'Our Service Levels' at the following link: https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-process", - "associatedMedia": "https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set,https://digital.nhs.uk/about-nhs-digital/corporate-information-and-documents/directions-and-data-provision-notices/data-provision-notices-dpns/improving-access-to-psychological-therapies-data-set-data-provision-notice", - "isPartOf": "mental health" + "associatedMedia": null, + "inPipeline": null }, "coverage": { "spatial": "United Kingdom,England", - "typicalAgeRange": "18-150", - "physicalSampleAvailability": null, "followup": "0 - 6 MONTHS", - "pathway": "" + "pathway": null, + "gender": null, + "typicalAgeRangeMin": 18, + "typicalAgeRangeMax": 150, + "datasetCompleteness": null, + "materialType": [ + "None/not available" + ] }, "provenance": { "origin": { - "purpose": "OTHER,ADMINISTRATIVE", - "source": "EPR", - "collectionSituation": "COMMUNITY,PRIMARY CARE" + "purpose": [ + "Other", + "Administrative" + ], + "source": [ + "EPR" + ], + "datasetType": "Health and disease", + "datasetSubType": "Not applicable", + "collectionSource": null, + "imageContrast": null }, "temporal": { - "accrualPeriodicity": "MONTHLY", "distributionReleaseDate": null, "startDate": "2012-04-01", "endDate": null, - "timeLag": "2-6 MONTHS" + "timeLag": "2-6 months", + "publishingFrequency": "Monthly" } }, "accessibility": { "usage": { - "dataUseLimitation": "NO RESTRICTION", - "dataUseRequirements": "INSTITUTION SPECIFIC RESTRICTIONS,PROJECT SPECIFIC RESTRICTIONS,TIME LIMIT ON USE", - "resourceCreator": "NHS DIGITIAL", - "investigations": "https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases", - "isReferencedBy": null + "dataUseLimitation": [ + "No restriction" + ], + "dataUseRequirements": null, + "resourceCreator": null }, "access": { "accessRights": "https://digital.nhs.uk/services/data-access-request-service-dars", @@ -61,67 +75,86 @@ "deliveryLeadTime": null, "jurisdiction": "GB-ENG", "dataProcessor": null, - "dataController": "NHS DIGITAL" + "dataController": "NHS DIGITAL", + "accessServiceCategory": null, + "accessMode": "New project" }, "formatAndStandards": { - "vocabularyEncodingScheme": "ODS,SNOMED CT,NHS NATIONAL CODES,ICD10", - "conformsTo": "NHS DATA DICTIONARY", + "vocabularyEncodingScheme": [ + "ODS", + "SNOMED CT", + "NHS NATIONAL CODES", + "ICD10" + ], + "conformsTo": [ + "NHS DATA DICTIONARY" + ], "language": "en", "format": "CSV" } }, "enrichmentAndLinkage": { - "qualifiedRelation": "https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases", - "derivation": "Data will be minimised as appropriate relative to the data access application", - "tools": "https://digital.nhs.uk/data-and-information/data-collections-and-data-sets/data-sets/improving-access-to-psychological-therapies-data-set/submitting-iapt-data" + "tools": null, + "derivedFrom": null, + "isPartOf": null, + "linkableDatasets": null, + "similarToDatasets": null, + "publicationAboutDataset": null, + "investigations": null, + "publicationUsingDataset": null }, "observations": [ { - "observedNode": "EVENTS", + "observedNode": "Event", "measuredValue": 114125, "measuredProperty": "COUNT", "observationDate": "2021-02-25", "disambiguatingDescription": "In December 2020 there were 114,125 referrals made into IAPT services in England." }, { - "observedNode": "EVENTS", + "observedNode": "Event", "measuredValue": 81578, "measuredProperty": "COUNT", "observationDate": "2021-02-25", "disambiguatingDescription": "In December 2020, 81,578 referrals into IAPT services in England started a course of treatment." }, { - "observedNode": "EVENTS", + "observedNode": "Event", "measuredValue": 48331, "measuredProperty": "COUNT", "observationDate": "2021-02-25", "disambiguatingDescription": "In December 2020, 48,331 referrals into IAPT services in England completed a course of treatment." } ], - "structuralMetadata": [ - { - "name": "IAPT.iapt.Rep_Referral", - "description": "IAPT.iapt.Rep_Referral", - "elements": [ - { - "name": "Count of number of Non-guided Self Help (Computer) sessions (derived)", - "description": "Count of number of Non-guided Self Help (Computer) sessions (derived)", - "dataType": "Number", - "sensitive": false - } - ] - }, - { - "name": "IAPT.iapt.Rep_Referral", - "description": "IAPT.iapt.Rep_Referral", - "elements": [ - { - "name": "Pseudonymised Service Request Identifier", - "description": "A request for the provision of care services to a PATIENT.", - "dataType": "String", - "sensitive": false - } - ] - } - ] -} + "structuralMetadata": { + "tables": [ + { + "name": "IAPT.iapt.Rep_Referral", + "description": "IAPT.iapt.Rep_Referral", + "columns": [ + { + "name": "Count of number of Non-guided Self Help (Computer) sessions (derived)", + "description": "Count of number of Non-guided Self Help (Computer) sessions (derived)", + "dataType": "Number", + "sensitive": false, + "values": null + } + ] + }, + { + "name": "IAPT.iapt.Rep_Referral", + "description": "IAPT.iapt.Rep_Referral", + "columns": [ + { + "name": "Pseudonymised Service Request Identifier", + "description": "A request for the provision of care services to a PATIENT.", + "dataType": "String", + "sensitive": false, + "values": null + } + ] + } + ], + "syntheticDataWebLink": null + } +} \ No newline at end of file