diff --git a/Tutorial.ipynb b/Tutorial.ipynb index 46e8d6e..96d0203 100644 --- a/Tutorial.ipynb +++ b/Tutorial.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 32, "id": "fa7b9741", "metadata": {}, "outputs": [], @@ -11,15 +11,14 @@ "import json\n", "\n", "\n", - "client_id = \"r9f9qDYkoRA2RwNu5Eni26pZhwxlFxinIosyfODL\"\n", - "app_id = \"wdG4zPgkFbP0CYStjx8tCpa4ncCSBaORcX0E4l8Z\"\n", - "api_path = \"https://api.dev.hdruk.cloud/api/v1\"\n", + "# client_id = \"r9f9qDYkoRA2RwNu5Eni26pZhwxlFxinIosyfODL\"\n", + "# app_id = \"wdG4zPgkFbP0CYStjx8tCpa4ncCSBaORcX0E4l8Z\"\n", + "# api_path = \"https://api.dev.hdruk.cloud/api/v1\"\n", "\n", "\n", - "#client_id = \"Yllt0NPHxBSxMXrdlegHU2ec0ltiNmyiOpl3FrFt\"\n", - "#app_id = \"KpG9cBeL7DTxnMciN5eZFQAknBYtkUmlIADIUVAM\"\n", - "#api_path = \"http://localhost:8000/api/v1\"\n", - "\n", + "# client_id = \"8X4Mq3U63db7Gn4LweZDml0PP2XmI5mF9P1K6JO9\"\n", + "# app_id = \"iEyMhfNc3t2chP3LpeiafCtrDSj5AWoYpnRbj0lY\"\n", + "# api_path = \"http://localhost:8000/api/v1\"\n", "\n", "headers = {\n", " \"x-client-id\": client_id,\n", @@ -32,7 +31,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 33, "id": "d28497d4", "metadata": {}, "outputs": [ @@ -42,8 +41,8 @@ "text": [ "{\n", " \"message\": \"created\",\n", - " \"data\": 26,\n", - " \"version\": 30\n", + " \"data\": 865,\n", + " \"version\": 869\n", "}\n" ] } @@ -57,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 34, "id": "8c59364c", "metadata": {}, "outputs": [ @@ -67,7 +66,7 @@ "201" ] }, - "execution_count": 3, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -78,17 +77,17 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 35, "id": "aeeebe94", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "26" + "865" ] }, - "execution_count": 4, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -100,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 36, "id": "193193ac", "metadata": {}, "outputs": [], @@ -113,17 +112,17 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 37, "id": "e65a292e", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "26" + "865" ] }, - "execution_count": 6, + "execution_count": 37, "metadata": {}, "output_type": "execute_result" } @@ -134,11 +133,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 38, "id": "f048faa2", - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -159,7 +156,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 39, "id": "887851fc", "metadata": {}, "outputs": [ @@ -178,10 +175,10 @@ " \"shortTitle\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"title\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"publisher\": {\n", - " \"gatewayId\": \"898f0f12-749c-4e2d-aa91-004533636c6b\",\n", - " \"name\": \"Dev Testing\"\n", + " \"gatewayId\": \"9e8f9c6f-373c-411e-be03-12c2d492e535\",\n", + " \"name\": \"Conn PLC\"\n", " },\n", - " \"populationSize\": 0,\n", + " \"populationSize\": -1,\n", " \"datasetSubType\": null\n", "}\n" ] @@ -193,7 +190,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 40, "id": "8f232012", "metadata": {}, "outputs": [ @@ -203,7 +200,7 @@ "['3.0.0']" ] }, - "execution_count": 9, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -216,17 +213,17 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 41, "id": "a2d3b278", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[30]" + "[869]" ] }, - "execution_count": 10, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -237,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 42, "id": "591b9b24", "metadata": {}, "outputs": [ @@ -263,14 +260,14 @@ ], "source": [ "response = requests.get(\n", - " f\"{api_path}/integrations/datasets/{dataset_id}?schema_model=HDRUK&schema_version=2.2.0\", headers=headers\n", + " f\"{api_path}/integrations/datasets/{dataset_id}?schema_model=HDRUK&schema_version=2.2.1\", headers=headers\n", ")\n", "print (json.dumps(response.json()['data']['coverage'],indent=6))" ] }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 43, "id": "a0874904", "metadata": {}, "outputs": [ @@ -282,9 +279,7 @@ " \"message\": \"failed to translate\",\n", " \"details\": {\n", " \"traser_message\": {\n", - " \"error\": \"Translation not found\",\n", - " \"message\": \"Request failed with status code 404\",\n", - " \"details\": \"Translation for GWDM-1.1 to HDRUK-2.2.3 is not implemented\"\n", + " \"message\": \"Cannot support the output model (HDRUK:2.2.3)\"\n", " },\n", " \"wasTranslated\": false,\n", " \"metadata\": null,\n", @@ -303,7 +298,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 44, "id": "9ae97f85", "metadata": {}, "outputs": [], @@ -317,7 +312,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 45, "id": "14be6979", "metadata": {}, "outputs": [ @@ -328,12 +323,12 @@ "{\n", " \"message\": \"success\",\n", " \"data\": {\n", - " \"id\": 9,\n", + " \"id\": 865,\n", " \"mongo_object_id\": null,\n", " \"mongo_id\": null,\n", " \"mongo_pid\": null,\n", " \"datasetid\": null,\n", - " \"pid\": \"ddb9ea45-39fe-4311-9273-f619de956bf9\",\n", + " \"pid\": \"d530e0f1-5da5-47b6-bb7b-fe3cda54db26\",\n", " \"source\": null,\n", " \"discourse_topic_id\": 0,\n", " \"is_cohort_discovery\": false,\n", @@ -341,34 +336,34 @@ " \"state_id\": 0,\n", " \"uploader_id\": 0,\n", " \"metadataquality_id\": 0,\n", - " \"user_id\": 48,\n", - " \"team_id\": 16,\n", + " \"user_id\": 3,\n", + " \"team_id\": 1,\n", " \"views_count\": 0,\n", " \"views_prev_count\": 0,\n", " \"has_technical_details\": 1,\n", - " \"created\": \"2024-02-22 10:15:52\",\n", - " \"updated\": \"2024-02-22 10:31:08\",\n", - " \"submitted\": \"2024-02-22 10:15:52\",\n", + " \"created\": \"2024-04-09 13:10:13\",\n", + " \"updated\": \"2024-04-09 13:10:49\",\n", + " \"submitted\": \"2024-04-09 13:10:13\",\n", " \"published\": null,\n", - " \"created_at\": \"2024-02-22T10:15:52.000000Z\",\n", - " \"updated_at\": \"2024-02-22T10:31:08.000000Z\",\n", + " \"created_at\": \"2024-04-09T13:10:13.000000Z\",\n", + " \"updated_at\": \"2024-04-09T13:10:49.000000Z\",\n", " \"deleted_at\": null,\n", " \"create_origin\": \"API\",\n", " \"status\": \"ACTIVE\",\n", " \"versions\": [\n", " {\n", - " \"id\": 9,\n", - " \"created_at\": \"2024-02-22T10:15:52.000000Z\",\n", - " \"updated_at\": \"2024-02-22T10:15:52.000000Z\",\n", + " \"id\": 869,\n", + " \"created_at\": \"2024-04-09T13:10:13.000000Z\",\n", + " \"updated_at\": \"2024-04-09T13:10:13.000000Z\",\n", " \"deleted_at\": null,\n", - " \"dataset_id\": 9,\n", + " \"dataset_id\": 865,\n", " \"metadata\": {\n", " \"metadata\": {\n", " \"required\": {\n", - " \"gatewayId\": \"9\",\n", - " \"gatewayPid\": \"ddb9ea45-39fe-4311-9273-f619de956bf9\",\n", - " \"issued\": \"2024-02-22T10:15:52.194758Z\",\n", - " \"modified\": \"2024-02-22T10:15:52.194774Z\",\n", + " \"gatewayId\": \"865\",\n", + " \"gatewayPid\": \"d530e0f1-5da5-47b6-bb7b-fe3cda54db26\",\n", + " \"issued\": \"2024-04-09T13:10:13.704312Z\",\n", + " \"modified\": \"2024-04-09T13:10:13.704317Z\",\n", " \"revisions\": [],\n", " \"version\": \"3.0.0\"\n", " },\n", @@ -383,10 +378,10 @@ " \"shortTitle\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"title\": \"Improving Access to Psychological Therapies Data Set\",\n", " \"publisher\": {\n", - " \"gatewayId\": \"efb50c2c-bb65-47d2-ac4b-ce6356f61b80\",\n", - " \"name\": \"Dev Testing\"\n", + " \"gatewayId\": \"9e8f9c6f-373c-411e-be03-12c2d492e535\",\n", + " \"name\": \"Conn PLC\"\n", " },\n", - " \"populationSize\": 0,\n", + " \"populationSize\": -1,\n", " \"datasetSubType\": null\n", " },\n", " \"coverage\": {\n", @@ -424,7 +419,8 @@ " \"dataProcessor\": null,\n", " \"accessRights\": \"https://digital.nhs.uk/services/data-access-request-service-dars\",\n", " \"accessService\": \"Once your DARS application has been approved, data will be made available either by secure file transfer or through the Data Access Environment (DAE). BL\\n\\nSecure file transfer: https://digital.nhs.uk/services/transfer-data-securely\\n\\nDAE: https://digital.nhs.uk/services/data-access-environment-dae\",\n", - " \"accessRequestCost\": \"https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-charges\"\n", + " \"accessRequestCost\": \"https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-charges\",\n", + " \"accessServiceCategory\": null\n", " },\n", " \"usage\": {\n", " \"dataUseLimitation\": \"NO RESTRICTION\",\n", @@ -454,7 +450,8 @@ " },\n", " \"investigations\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", " \"isGeneratedUsing\": null,\n", - " \"dataUses\": null\n", + " \"dataUses\": null,\n", + " \"syntheticDataWebLink\": null\n", " },\n", " \"observations\": [\n", " {\n", @@ -651,26 +648,26 @@ " }\n", " ]\n", " },\n", - " \"gwdmVersion\": \"1.1\"\n", + " \"gwdmVersion\": \"1.2\"\n", " },\n", " \"version\": 1\n", " },\n", " {\n", - " \"id\": 10,\n", - " \"created_at\": \"2024-02-22T10:31:08.000000Z\",\n", - " \"updated_at\": \"2024-02-22T10:31:08.000000Z\",\n", + " \"id\": 870,\n", + " \"created_at\": \"2024-04-09T13:10:49.000000Z\",\n", + " \"updated_at\": \"2024-04-09T13:10:49.000000Z\",\n", " \"deleted_at\": null,\n", - " \"dataset_id\": 9,\n", + " \"dataset_id\": 865,\n", " \"metadata\": {\n", " \"metadata\": {\n", " \"required\": {\n", - " \"gatewayId\": \"9\",\n", - " \"gatewayPid\": \"ddb9ea45-39fe-4311-9273-f619de956bf9\",\n", + " \"gatewayId\": \"865\",\n", + " \"gatewayPid\": \"d530e0f1-5da5-47b6-bb7b-fe3cda54db26\",\n", " \"issued\": \"2021-05-10T00:00:00.000Z\",\n", - " \"modified\": \"2024-02-22T10:31:08.505902Z\",\n", + " \"modified\": \"2024-04-09T13:10:49.928091Z\",\n", " \"revisions\": [\n", " {\n", - " \"url\": \"https://placeholder.blah/ddb9ea45-39fe-4311-9273-f619de956bf9?version=3.0.0\",\n", + " \"url\": \"https://placeholder.blah/d530e0f1-5da5-47b6-bb7b-fe3cda54db26?version=3.0.0\",\n", " \"version\": \"3.0.0\"\n", " }\n", " ],\n", @@ -687,11 +684,11 @@ " \"shortTitle\": \"UPDATED TITLE\",\n", " \"title\": \"UPDATED TITLE\",\n", " \"publisher\": {\n", - " \"name\": \"Dev Testing\",\n", - " \"gatewayId\": \"efb50c2c-bb65-47d2-ac4b-ce6356f61b80\",\n", + " \"name\": \"Conn PLC\",\n", + " \"gatewayId\": \"9e8f9c6f-373c-411e-be03-12c2d492e535\",\n", " \"rorId\": null\n", " },\n", - " \"populationSize\": 0,\n", + " \"populationSize\": -1,\n", " \"datasetSubType\": null\n", " },\n", " \"coverage\": {\n", @@ -729,7 +726,8 @@ " \"dataProcessor\": null,\n", " \"accessRights\": \"https://digital.nhs.uk/services/data-access-request-service-dars\",\n", " \"accessService\": \"Once your DARS application has been approved, data will be made available either by secure file transfer or through the Data Access Environment (DAE). BL\\n\\nSecure file transfer: https://digital.nhs.uk/services/transfer-data-securely\\n\\nDAE: https://digital.nhs.uk/services/data-access-environment-dae\",\n", - " \"accessRequestCost\": \"https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-charges\"\n", + " \"accessRequestCost\": \"https://digital.nhs.uk/services/data-access-request-service-dars/data-access-request-service-dars-charges\",\n", + " \"accessServiceCategory\": null\n", " },\n", " \"usage\": {\n", " \"dataUseLimitation\": \"NO RESTRICTION\",\n", @@ -759,7 +757,8 @@ " },\n", " \"investigations\": \"https://digital.nhs.uk/services/data-access-request-service-dars/register-of-approved-data-releases\",\n", " \"isGeneratedUsing\": null,\n", - " \"dataUses\": null\n", + " \"dataUses\": null,\n", + " \"syntheticDataWebLink\": null\n", " },\n", " \"observations\": [\n", " {\n", @@ -829,7 +828,7 @@ " \"app\": {\n", " \"id\": 0\n", " },\n", - " \"id\": \"9\",\n", + " \"id\": \"865\",\n", " \"original_metadata\": {\n", " \"identifier\": \"https://web.www.healthdatagateway.org/19525c5f-92ee-41b6-bb79-673624b27bdd\",\n", " \"version\": \"3.0.1\",\n", @@ -957,7 +956,7 @@ " }\n", " ]\n", " },\n", - " \"gwdmVersion\": \"1.1\"\n", + " \"gwdmVersion\": \"1.2\"\n", " },\n", " \"version\": 2\n", " }\n", @@ -976,7 +975,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 46, "id": "dfce3deb", "metadata": {}, "outputs": [], @@ -988,7 +987,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 47, "id": "e75312a6", "metadata": {}, "outputs": [ @@ -998,7 +997,7 @@ "['3.0.0', '3.0.1']" ] }, - "execution_count": 113, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -1013,7 +1012,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 48, "id": "b8544866", "metadata": {}, "outputs": [], @@ -1026,7 +1025,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 49, "id": "ed8402dd", "metadata": {}, "outputs": [ @@ -1036,7 +1035,7 @@ "{'message': 'success'}" ] }, - "execution_count": 115, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -1070,7 +1069,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.12.2" } }, "nbformat": 4, diff --git a/example-gwdm10.json b/example-gwdm12.json similarity index 81% rename from example-gwdm10.json rename to example-gwdm12.json index f17aa88..d5262dc 100644 --- a/example-gwdm10.json +++ b/example-gwdm12.json @@ -21,29 +21,39 @@ "version": "2.1.1", "url": "https://a7ddefbd-31d9-4703-a738-256e4689f76a" } - ] + ], + "version": "0.0.1" }, "summary": { "abstract": "CALUM DATASET TEST", "contactPoint": "susheel.varma@hdruk.ac.uk", "keywords": "Preprints,Papers,HDR UK", - "controlledKeywords": "", + "controlledKeywords": null, "datasetType": "list of papers", "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", "doiName": "10.1093/ije/dyx196", "shortTitle": "HDR UK Papers & Preprints", "title": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", "publisher": { - "publisherName": "BREATHE", - "publisherGatewayId": "1234" - } + "name": "BREATHE", + "gatewayId": "1234", + "rorId": null + }, + "populationSize": 907515, + "datasetSubType": null }, "coverage": { "pathway": "NOT APPLICABLE", - "physicalSampleAvailability": "NOT AVAILABLE", "spatial": "https://www.geonames.org/countries/GB/united-kingdom.html", "followup": "UNKNOWN", - "typicalAgeRange": "0-0" + "typicalAgeRange": "0-0", + "biologicalsamples": null, + "gender": null, + "psychological": null, + "physical": null, + "anthropometric": null, + "lifestyle": null, + "socioeconomic": null }, "provenance": { "origin": { @@ -67,12 +77,17 @@ "dataProcessor": "HDR UK", "accessRights": "https://raw.githubusercontent.com/HDRUK/papers/master/LICENSE", "accessService": "https://github.com/HDRUK/papers", - "accessRequestCost": "Free" + "accessRequestCost": "Free", + "accessServiceCategory": null }, "usage": { "dataUseLimitation": "GENERAL RESEARCH USE", "dataUseRequirement": "RETURN TO DATABASE OR RESOURCE", - "resourceCreator": "HDR UK Science Team" + "resourceCreator": { + "name": "HDR UK Science Team", + "gatewayId": null, + "rorId": null + } }, "formatAndStandards": { "vocabularyEncodingSchemes": "OTHER", @@ -92,7 +107,9 @@ "isMemberOf": "blah", "linkedDatasets": "https://web.www.healthdatagateway.org/dataset/fd8d0743-344a-4758-bb97-f8ad84a37357" }, - "investigations": "https://github.com/HDRUK/papers" + "investigations": "https://github.com/HDRUK/papers", + "associatedMedia": null, + "syntheticDataWebLink": null }, "observations": [ { @@ -119,9 +136,11 @@ "name": "column1", "description": "this is column1", "dataType": "String", - "sensitive": false + "sensitive": false, + "values": null } ] } - ] -} + ], + "tissuesSampleCollection": null +} \ No newline at end of file