From 94401904f2d4496902eb2247678de4d01a9e521d Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Fri, 16 Feb 2024 10:11:06 +0000 Subject: [PATCH 1/8] add some more exemplar testing --- .../1460e2b4-a985-4890-8e60-a21e78ce01f3.json | 142 +++++ .../77dc7118-443c-4c85-a18c-e7bbe01e9e7a.json | 581 ++++++++++++++++++ ...d9ff-0d54-4438-9c60-423848aebb4c.json.json | 185 ++++++ files/exemplar/list.json | 46 ++ src/api/v1/controllers/dataset.controller.ts | 42 +- src/api/v1/routes/dataset.route.ts | 3 + src/api/v1/services/dataset.service.ts | 26 +- src/server.ts | 2 +- 8 files changed, 1019 insertions(+), 8 deletions(-) create mode 100644 files/exemplar/1460e2b4-a985-4890-8e60-a21e78ce01f3.json create mode 100644 files/exemplar/77dc7118-443c-4c85-a18c-e7bbe01e9e7a.json create mode 100644 files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json.json create mode 100644 files/exemplar/list.json diff --git a/files/exemplar/1460e2b4-a985-4890-8e60-a21e78ce01f3.json b/files/exemplar/1460e2b4-a985-4890-8e60-a21e78ce01f3.json new file mode 100644 index 0000000..d33e386 --- /dev/null +++ b/files/exemplar/1460e2b4-a985-4890-8e60-a21e78ce01f3.json @@ -0,0 +1,142 @@ +{ + "identifier": "https://web.www.healthdatagateway.org/1460e2b4-a985-4890-8e60-a21e78ce01f3", + "version": "2.0.0", + "issued": "2022-04-19T00:00:00.000Z", + "modified": "2022-04-19T00:00:00.000Z", + "revisions": [], + "summary": { + "title": "UK Biobank", + "abstract": "UK Biobank is a large-scale biomedical database and research resource that provides researchers access to detailed longitudinal phenotype, medical and genetic data from 500,000 volunteer participants. ", + "publisher": { + "identifier": "https://web.www.healthdatagateway.org/607db9c6e1f9d3704d570d7f", + "name": "UK Biobank", + "logo": null, + "description": null, + "contactPoint": null, + "memberOf": "ALLIANCE" + }, + "contactPoint": "access@ukbiobank.ac.uk", + "keywords": "UK BIOBANK,Genomics,Exome sequencing,WGS,Omics,Pain,Research,Cognitive Measures,Physical Measures,Magnetic resonance imaging,DXA,ECG,Accelerometer,Mental Health,Environment,Primary Care,COVID-19,Hospital episode statistics,Cancer Registry,Deaths,Sociodemographics,Digestive Health,Occupational Health,Biomarkers,Lifestyle,Health Data,Cardiac MRI,Brain MRI,Abdominal MRI,Carotid Ultrasound,Diet,Pain Hub", + "alternateIdentifiers": null, + "doiName": null, + "datasetType": "Biobank", + "datasetSubType": "Biobank", + "populationSize": 500000 + }, + "documentation": { + "description": "UK Biobank is a large-scale biomedical database and research resource, containing in-depth genetic and health information from half a million UK participants. The database, which is regularly augmented with additional data, is globally accessible to approved researchers and scientists undertaking vital research into the most common and life-threatening diseases. UK Biobank’s research resource is a major contributor to the advancement of modern medicine and treatment and has enabled several scientific discoveries that improve human health.\n\nSince 2006, UK Biobank has collected an unprecedented amount of biological and medical data on half a million people, aged between 40 and 69 years old and living in the UK, as part of a large-scale prospective study. With their consent they regularly provide blood, urine and saliva samples, as well as detailed information about their lifestyle which is then linked to their health-related records to provide a deeper understanding of how individuals experience diseases. Genotyping, whole exome sequencing and whole genome sequencing is available for the whole cohort. Blood and urine biomarkers, telomere data, metabolomic and proteomic data and infectious disease markers have been assayed from the samples provided.\n\nSince 2014 we have been undertaking the largest imaging study to date. We aim to undertake brain, cardiac and neck to knee MRI, whole body DXA and carotid ultrasound of 100,000 participants. We additionally have retinal images for 100,000 participants from baseline assessment, and accelerometer data for 100,000 participants collected 2013-2014.\n\nQuestionnaires that aim to capture data that is not readily captured by health data linkages are regularly sent to our participants.\n\nThe data – the largest and richest dataset of its kind – is de-identified and made widely accessible by UK Biobank to registered researchers around the world who use it to make new scientific discoveries about common and life-threatening diseases – such as cancer, heart disease and stroke – in order to improve public health.", + "associatedMedia": "https://biobank.ndph.ox.ac.uk/showcase/index.cgi,https://www.ukbiobank.ac.uk/", + "isPartOf": null + }, + "coverage": { + "spatial": "United Kingdom", + "typicalAgeRange": "40-69", + "followup": "CONTINUOUS", + "pathway": "UK Biobank is a volunteer based cohort. As such, there is a healthy volunteer effect that results in participants tending to be of higher socioeconomic status, remaining in education longer, slimmer, less smokers (although those that smoke tend to be heavier smokers) and lower consumers of alcohol than the general population. A comparison between UK Biobank participants and the general UK population has been published (https://doi.org/10.1093/aje/kwx246).\n\nWhilst selection biases are seen in UK Biobank, there is still substantial heterogeneity within the cohort. Whilst incidence and prevalence calculations are not generalisable to the UK population, exposure-outcome comparisons should be due to the heterogeneity in the cohort. However, it is important that researchers consider the potential biases of a data set that might limit generalisability of their results (as is the case for all observational data). ", + "biologicalsamples": ["Saliva", "Urine"], + "gender": null, + "psychological": null, + "physical": null, + "anthropometric": null, + "lifestyle": null, + "socioeconomic": null + }, + "provenance": { + "origin": { + "purpose": "STUDY", + "source": "EPR,ELECTRONIC SURVEY,MACHINE GENERATED", + "collectionSituation": "PRIMARY CARE,ACCIDENT AND EMERGENCY,IN-PATIENTS,COMMUNITY,CLINIC,PHARMACY" + }, + "temporal": { + "distributionReleaseDate": null, + "startDate": "2006-03-13", + "endDate": null, + "timeLag": "VARIABLE", + "publishingFrequency": "CONTINUOUS" + } + }, + "accessibility": { + "usage": { + "dataUseLimitation": "GENERAL RESEARCH USE", + "dataUseRequirements": "INSTITUTION SPECIFIC RESTRICTIONS,PROJECT SPECIFIC RESTRICTIONS,PUBLICATION REQUIRED,RETURN TO DATABASE OR RESOURCE,USER SPECIFIC RESTRICTION,TIME LIMIT ON USE", + "resourceCreator": "UK Biobank", + "investigations": "https://www.ukbiobank.ac.uk/enable-your-research/approved-research", + "isReferencedBy": "Sudlow C, Gallacher J, Allen N, Beral V, Burton P, Danesh J, et al. (2015) UK Biobank: An Open Access Resource for Identifying the Causes of a Wide Range of Complex Diseases of Middle and Old Age. PLoS Med 12(3): e1001779. https://doi.org/10.1371/journal.pmed.1001779,Bycroft, C., Freeman, C., Petkova, D. et al. The UK Biobank resource with deep phenotyping and genomic data. Nature 562, 203–209 (2018). https://doi.org/10.1038/s41586-018-0579-z,Conroy M, Sellors J, Effingham M, et al. The advantages of UK Biobank’s open-access strategy for health research. J Intern Med. 2019;286(4):389-397. doi:10.1111/joim.12955,Littlejohns TJ, Holliday J, Gibson LM, et al. The UK Biobank imaging enhancement of 100,000 participants: rationale, data collection, management and future directions. Nat Commun. 2020;11(1):2624. doi:10.1038/s41467-020-15948-9,Fry A, Littlejohns TJ, Sudlow C, et al. Comparison of Sociodemographic and Health-Related Characteristics of UK Biobank Participants With Those of the General Population. Am J Epidemiol. 2017:1-9. doi:10.1093/aje/kwx246" + }, + "access": { + "accessRights": "https://www.ukbiobank.ac.uk/enable-your-research/apply-for-access", + "accessService": "Applications to access data are made through our bespoke access management system (https://bbams.ndph.ox.ac.uk/ams/). \n\nData access is either via data download (phenotype and genotype data) or via our Research Analysis Platform (phenotype, imaging, genotype, WES, WGS, omics). Our RAP is enabled by DNANexus and hosted by Amazon Web Services (https://www.ukbiobank.ac.uk/enable-your-research/research-analysis-platform). \n\nAccess costs depend on what data access is required.", + "accessRequestCost": "https://www.ukbiobank.ac.uk/enable-your-research/costs", + "deliveryLeadTime": null, + "jurisdiction": "GB-ENG", + "dataProcessor": "UK Biobank", + "dataController": "UK Biobank" + }, + "formatAndStandards": { + "vocabularyEncodingScheme": "LOCAL,OPCS4,READ,SNOMED CT,DM+D,ICD10,ICD9", + "conformsTo": "DICOM,LOCAL,NHS DATA DICTIONARY", + "language": "en", + "format": "Text/csv, dta, SAS, R,Image/ DICOM, NIFTI, PNG,Other/ VCF, CRAM, PLINK, BGEN, BED, CWA" + } + }, + "enrichmentAndLinkage": { + "qualifiedRelation": null, + "derivation": null, + "tools": null + }, + "observations": [ + { + "observedNode": "PERSONS", + "measuredValue": 500000, + "observationDate": "2006-03-13", + "measuredProperty": "Count", + "disambiguatingDescription": "Each participant has a large number (<5000) of data points associated with them. Recruitment started in 2006, but data collection is ongoing, and health data predates recruitment date. Summary statistics of all data can be found on our data showcase." + } + ], + "structuralMetadata": [ + { + "name": "UK Biobank Data Dictionary", + "elements": [ + { + "name": 3, + "description": "Verbal interview duration", + "dataType": "Integer", + "sensitive": false + } + ], + "description": null + } + ], + "tissuesSampleCollection": [ + { + "materialType": ["Serum"], + "dataCategories": null, + "tissueSampleMetadata": null, + "collectionType": null + }, + { + "materialType": ["Plasma"], + "dataCategories": null, + "tissueSampleMetadata": null, + "collectionType": null + }, + { + "materialType": ["Blood"], + "dataCategories": null, + "tissueSampleMetadata": null, + "collectionType": null + }, + { + "materialType": ["Saliva"], + "dataCategories": null, + "tissueSampleMetadata": null, + "collectionType": null + }, + { + "materialType": ["Urine"], + "dataCategories": null, + "tissueSampleMetadata": null, + "collectionType": null + } + ] +} diff --git a/files/exemplar/77dc7118-443c-4c85-a18c-e7bbe01e9e7a.json b/files/exemplar/77dc7118-443c-4c85-a18c-e7bbe01e9e7a.json new file mode 100644 index 0000000..0f5a8c5 --- /dev/null +++ b/files/exemplar/77dc7118-443c-4c85-a18c-e7bbe01e9e7a.json @@ -0,0 +1,581 @@ +{ + "identifier": "https://web.www.healthdatagateway.org/77dc7118-443c-4c85-a18c-e7bbe01e9e7a", + "version": "1.0.0", + "issued": "2021-09-23T00:00:00.000Z", + "modified": "2021-09-23T00:00:00.000Z", + "revisions": [], + "summary": { + "title": "1958 National Child Development Study - Pain data", + "abstract": "This collection specifically relates to a short pain survey (self-completion booklet) completed by NCDS participants in 2002-3.", + "publisher": { + "identifier": "https://web.www.healthdatagateway.org/613b2174b8b58d157f531e7a", + "name": "ALLEVIATE", + "logo": null, + "description": null, + "contactPoint": null, + "memberOf": "HUB" + }, + "contactPoint": "blah@blah.com", + "keywords": "Alleviate,Pain,Pain Hub", + "alternateIdentifiers": null, + "doiName": "10.5255/UKDA-SN-8731-1" + }, + "documentation": { + "description": "The 1958 National Child Development Study (NCDS) is following the lives of an initial 17,415 people born in England, Scotland and Wales in a single week of 1958. It started in 1958 at birth, as the Perinatal Mortality Survey. Over the course of cohort members\u2019 lives, information has been collected on their physical and educational development, economic circumstances, employment, family life, health behaviour, wellbeing, social participation and attitudes.\n\nThis collection specifically relates to a short pain survey (self-completion booklet) completed by NCDS participants in 2002-3.", + "associatedMedia": "https://discovery.closer.ac.uk/item/uk.cls.ncds/107fe7d5-b871-463e-853c-a592552b591a/1", + "isPartOf": "https://cls.ucl.ac.uk/cls-studies/1958-national-child-development-study-2/" + }, + "coverage": { + "spatial": "https://www.geonames.org/2648147/great-britain.html", + "typicalAgeRange": "31-59", + "physicalSampleAvailability": null, + "followup": "> 10 YEARS", + "pathway": "Data are not associated with a patient pathway." + }, + "provenance": { + "origin": { + "purpose": "STUDY", + "source": "PAPER BASED", + "collectionSituation": "COMMUNITY,HOME" + }, + "temporal": { + "accrualPeriodicity": "STATIC", + "distributionReleaseDate": "2004-03-31", + "startDate": "2002-09-01", + "endDate": "2004-03-31", + "timeLag": "1-2 MONTHS" + } + }, + "accessibility": { + "usage": { + "dataUseLimitation": "GENERAL RESEARCH USE", + "dataUseRequirements": "PROJECT SPECIFIC RESTRICTIONS", + "resourceCreator": "UK Data Services", + "investigations": "https://www.ukri.org/news/new-data-hub-and-research-into-chronic-pain/", + "isReferencedBy": "http://repec.ioe.ac.uk/REPEc/pdf/qsswp2128.pdf" + }, + "access": { + "accessRights": "https://beta.ukdataservice.ac.uk/datacatalogue/studies/study?id=8731#!/access-data", + "accessService": "The Data Collection is available to users registered with the UK Data Service.\n\nCommercial use of the data requires approval from the data owner or their nominee. The UK Data Service will contact you.", + "accessRequestCost": "https://beta.ukdataservice.ac.uk/datacatalogue/studies/study?id=8731#!/access-data", + "deliveryLeadTime": null, + "jurisdiction": "GB-GB", + "dataProcessor": "George Ploubidis. Professor of Population Health and Statistics at the UCL Social Research Institute.", + "dataController": "University College London is the Data Controller and is committed to protecting the rights of individuals in line with the Data Protection Act 2018 (DPA) and the new General Data Protection Regulation (GDPR).\n\nYour test result and unique ID will be retained by Thriva for three years. No other information will be retained by Thriva. The Department for Health and Social Care are the Data Controller for the data that will be retained by Thriva." + }, + "formatAndStandards": { + "vocabularyEncodingScheme": "NHS NATIONAL CODES,OTHER,LOCAL", + "conformsTo": "NHS DATA DICTIONARY", + "language": "en", + "format": "Text" + } + }, + "enrichmentAndLinkage": { + "qualifiedRelation": "https://cls.ucl.ac.uk/data_documentation/?studies%5B%5D=age-42-sweep&studies%5B%5D=substudies&studies%5B%5D=ncds-birth-sweep&studies%5B%5D=age-7-sweep&studies%5B%5D=age-11-sweep&studies%5B%5D=age-16-sweep&studies%5B%5D=age-23-sweep&studies%5B%5D=age-33-sweep&studies%5B%5D=age-44-biomedical-sweep&studies%5B%5D=age-46-sweep&studies%5B%5D=age-50-sweep&studies%5B%5D=age-55-sweep&studies%5B%5D=age-62-sweep&s=&filter=1", + "derivation": "Not Known", + "tools": null + }, + "observations": [ + { + "observedNode": "FINDINGS", + "measuredValue": 8565, + "observationDate": "2002-09-01", + "measuredProperty": "Count", + "disambiguatingDescription": "Self-reported pain questionnaire" + } + ], + "structuralMetadata": [ + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "BE1", + "description": "SC1 (E1): Pain during the last month lasting > one day", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "BE2", + "description": "SC1 (E2): Have you been aware of this pain for > 3 months", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR1", + "description": "SC1 (E3): ACRU Pain Definition - Left Upper", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR2", + "description": "SC1 (E3): ACRU Pain Definition - Right Upper", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR3", + "description": "SC1 (E3): ACRU Pain Definition - Left Lower", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR4", + "description": "SC1 (E3): ACRU Pain Definition - Right Lower", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR5", + "description": "SC1 (E3): ACRU Pain Definition - Upper Spine", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR6", + "description": "SC1 (E3): ACRU Pain Definition - Left: Low Back", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR7", + "description": "SC1 (E3): ACRU Pain Definition - Lower Spine", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR8", + "description": "SC1 (E3): ACRU Pain Definition - Right: Low Back", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR9", + "description": "SC1 (E3): ACRU Pain Definition - Left Buttock", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_ACR10", + "description": "SC1 (E3): ACRU Pain Definition - Right Buttock", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN1", + "description": "SC1 (E3): Manchester Pain Definition - Right Shoulder", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN2", + "description": "SC1 (E3): Manchester Pain Definition - Right Elbow", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN3", + "description": "SC1 (E3): Manchester Pain Definition - Right Forearm", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN4", + "description": "SC1 (E3): Manchester Pain Definition - Right Hand", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN5", + "description": "SC1 (E3): Manchester Pain Definition - Left Shoulder", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN6", + "description": "SC1 (E3): Manchester Pain Definition - Left Elbow", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN7", + "description": "SC1 (E3): Manchester Pain Definition - Left Forearm", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN8", + "description": "SC1 (E3): Manchester Pain Definition - Left Hand", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN9", + "description": "SC1 (E3): Manchester Pain Definition - Right Thigh", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN10", + "description": "SC1 (E3): Manchester Pain Definition - Right Knee", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN11", + "description": "SC1 (E3): Manchester Pain Definition - Right Lower Leg", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN12", + "description": "SC1 (E3): Manchester Pain Definition - Right Foot", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN13", + "description": "SC1 (E3): Manchester Pain Definition - Left Thigh", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN14", + "description": "SC1 (E3): Manchester Pain Definition - Left Knee", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN15", + "description": "SC1 (E3): Manchester Pain Definition - Left Lower Leg", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN16", + "description": "SC1 (E3): Manchester Pain Definition - Left Foot", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN17", + "description": "SC1 (E3): Manchester Pain Definition - Head", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN18", + "description": "SC1 (E3): Manchester Pain Definition - Throat / Sternum", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN19", + "description": "SC1 (E3): Manchester Pain Definition - Right Chest", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN20", + "description": "SC1 (E3): Manchester Pain Definition - Left Chest", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN21", + "description": "SC1 (E3): Manchester Pain Definition - Abdomen", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN22", + "description": "SC1 (E3): Manchester Pain Definition - Left Upper Back", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN23", + "description": "SC1 (E3): Manchester Pain Definition - Right Upper Back", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN24", + "description": "SC1 (E3): Manchester Pain Definition - Upper Spine", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN25", + "description": "SC1 (E3): Manchester Pain Definition - Left Low Back", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN26", + "description": "SC1 (E3): Manchester Pain Definition - Right Low Back", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN27", + "description": "SC1 (E3): Manchester Pain Definition - Lower Spine", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN28", + "description": "SC1 (E3): Manchester Pain Definition - Left Buttock", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + }, + { + "name": "NCDS Biomedical Paper Self-Completion 1 (2002) Dataset (85 of 172)", + "elements": [ + { + "name": "E3_MAN29", + "description": "SC1 (E3): Manchester Pain Definition - Right Buttock", + "dataType": "Frequency", + "sensitive": false + } + ], + "description": null + } + ] +} diff --git a/files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json.json b/files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json.json new file mode 100644 index 0000000..017218b --- /dev/null +++ b/files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json.json @@ -0,0 +1,185 @@ +{ + "identifier": "https://blah.com/7eb5d9ff-0d54-4438-9c60-423848aebb4c", + "version": "11.0.0", + "issued": "2023-08-17T00:00:00.000Z", + "modified": "2023-08-17T00:00:00.000Z", + "revisions": [], + "summary": { + "title": "Education Daily Attendance Dataset (EDAD)", + "abstract": "This dataset provides information about daily educational attendance within Wales.", + "publisher": { + "identifier": "https://blah.com/5f3f98068af2ef61552e1d75", + "name": "SAIL", + "logo": null, + "description": null, + "contactPoint": null, + "memberOf": "ALLIANCE" + }, + "contactPoint": "SAILDatabank@blah.com", + "keywords": "Education, SAIL,EDAD,core-res,Health & Wellbeing,Children & Young People,Inequality & Social Inclusion, Housing & Communities", + "alternateIdentifiers": null, + "doiName": null + }, + "documentation": { + "description": "This dataset provides detailed information about daily educational attendance within Wales.\n\nAttendance data in the EDUW schema was discontinued after 2019 and the Education Daily Attendance Dataset (EDAD) schema replaced it. This dataset contains more detailed information on attendance than was previously available in EDUW.\n\nThere is no ALF within this dataset, projects will need apply for education (EDUW) data to enable ALF linkage to other datasets", + "associatedMedia": "https://saildatabank.com/about-us/", + "isPartOf": null + }, + "coverage": { + "spatial": "United Kingdom,Wales", + "typicalAgeRange": "0-20", + "physicalSampleAvailability": null, + "followup": "0 - 6 MONTHS", + "pathway": "Education" + }, + "provenance": { + "origin": { + "purpose": "ADMINISTRATIVE", + "source": "OTHER", + "collectionSituation": "OTHER" + }, + "temporal": { + "accrualPeriodicity": "MONTHLY", + "distributionReleaseDate": null, + "startDate": "2020-09-01", + "endDate": "2021-12-17", + "timeLag": "2-6 MONTHS" + } + }, + "accessibility": { + "usage": { + "dataUseLimitation": "GENERAL RESEARCH USE", + "dataUseRequirements": "PROJECT SPECIFIC RESTRICTIONS,USER SPECIFIC RESTRICTION", + "resourceCreator": "Welsh Government", + "investigations": null, + "isReferencedBy": "Littlecott HJ, Moore GF, Moore L, Lyons RA, Murphy S. Association between breakfast consumption and educational outcomes in 9-11-year-old children. Public Health Nutr. 2016 Jun;19(9):1575-82. doi: 10.1017/S1368980015002669. Epub 2015 Sep 28. Erratum in: Public Health Nutr. 2016 Jun;19(9):1583. PMID: 26411331; PMCID: PMC4873891. " + }, + "access": { + "accessRights": "https://saildatabank.com/data/apply-to-work-with-the-data/", + "accessService": "The SAIL Databank is powered by the UK Secure e-Research Platform (UKSeRP). Following approval through safeguard processes, access to project-specific data within the secure environment is permitted using two-factor authentication.", + "accessRequestCost": "Data provision is free from SAIL. Overall project costing depends on the number of people that require access to the SAIL Gateway, the activities that SAIL needs to complete (e.g. loading non-standard datasets), data refreshes, analytical work required, disclosure control process, and special case technological requirements.", + "deliveryLeadTime": null, + "jurisdiction": "GB", + "dataProcessor": "SAIL Databank", + "dataController": "Welsh Government" + }, + "formatAndStandards": { + "vocabularyEncodingScheme": "LOCAL", + "conformsTo": "LOCAL", + "language": "en", + "format": "SQL database table" + } + }, + "enrichmentAndLinkage": { + "qualifiedRelation": "Yes. To any SAIL dataset & reference data.", + "derivation": null, + "tools": "https://conceptlibrary.saildatabank.com/" + }, + "observations": [ + { + "observedNode": "EVENTS", + "measuredValue": 3000000, + "observationDate": "2021-09-01", + "measuredProperty": "Count", + "disambiguatingDescription": "Approx 70-80% coverage of Welsh population through GP registrations." + } + ], + "structuralMetadata": [ + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "AVAIL_FROM_DT", + "description": "Date when the data made available i.e. date of loading", + "dataType": "DATE", + "sensitive": false + } + ] + }, + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "DATEATTENDANCE", + "description": "Date of session attendance", + "dataType": "DATE", + "sensitive": false + } + ] + }, + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "ESTAB_E", + "description": "Establishment Code - Encrypted", + "dataType": "BIGINT", + "sensitive": false + } + ] + }, + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "LEA", + "description": "Local Education Area", + "dataType": "INTEGER", + "sensitive": false + } + ] + }, + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "SCHOOLREF_E", + "description": "School Reference ID", + "dataType": "BIGINT", + "sensitive": false + } + ] + }, + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "SESSION", + "description": "-", + "dataType": "VARCHAR", + "sensitive": false + } + ] + }, + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "SESSIONTYPE", + "description": "To idenitify whether the session was in the morning or afternoon e.g. AM", + "dataType": "VARCHAR", + "sensitive": false + } + ] + }, + { + "name": "EDAD", + "description": "Education Daily Activity Dataset table", + "elements": [ + { + "name": "SYSTEM_ID_E", + "description": "Local pupil identifier. This is specific to the school and one ALF may have mulitple SYSTEM_ID", + "dataType": "BIGINT", + "sensitive": false + } + ] + } + ] +} diff --git a/files/exemplar/list.json b/files/exemplar/list.json new file mode 100644 index 0000000..40889be --- /dev/null +++ b/files/exemplar/list.json @@ -0,0 +1,46 @@ +{ + "items": [ + { + "name": "SAIL EDAD", + "@schema": "https://raw.githubusercontent.com/HDRUK/schemata-2/master/hdr_schemata/models/HDRUK/2.1.2/schema.json", + "description": "Examplar SAIL dataset Education Daily Attendance Dataset (EDAD)", + "type": "dataset", + "persistentId": "7eb5d9ff-0d54-4438-9c60-423848aebb4c", + "self": "http://example-url.com/api/datasets/7eb5d9ff-0d54-4438-9c60-423848aebb4c", + "version": "11.0.0", + "issued": "2023-08-17T00:00:00.000Z", + "modified": "2023-08-17T00:00:00.000Z", + "source": "SAIL" + }, + { + "name": "Alleviate 1958 National Child Development Study ", + "@schema": "https://raw.githubusercontent.com/HDRUK/schemata-2/master/hdr_schemata/models/HDRUK/2.1.2/schema.json", + "description": "Examplar Alleviate dataset", + "type": "dataset", + "persistentId": "77dc7118-443c-4c85-a18c-e7bbe01e9e7a", + "self": "http://example-url.com/api/datasets/77dc7118-443c-4c85-a18c-e7bbe01e9e7a", + "version": "1.0.0", + "issued": "2021-09-23T00:00:00.000Z", + "modified": "2021-09-23T00:00:00.000Z", + "source": "ALLEVIATE" + }, + { + "name": "UKBiobank with tissues", + "@schema": "https://raw.githubusercontent.com/HDRUK/schemata-2/master/hdr_schemata/models/HDRUK/2.1.2/schema.json", + "description": "Examplar UKBiobank", + "type": "dataset", + "persistentId": "1460e2b4-a985-4890-8e60-a21e78ce01f3", + "self": "http://example-url.com/api/datasets/1460e2b4-a985-4890-8e60-a21e78ce01f3", + "version": "11.0.0", + "issued": "2022-04-19T00:00:00.000Z", + "modified": "2022-04-19T00:00:00.000Z", + "source": "UKBiobank" + } + ], + "query": { + "q": "", + "total": 3, + "limit": 0, + "offset": 0 + } +} diff --git a/src/api/v1/controllers/dataset.controller.ts b/src/api/v1/controllers/dataset.controller.ts index b072d39..311c3dd 100644 --- a/src/api/v1/controllers/dataset.controller.ts +++ b/src/api/v1/controllers/dataset.controller.ts @@ -13,20 +13,21 @@ export default class DatasetController extends BaseController { this.getDatasets = this.getDatasets.bind(this); this.getDataset = this.getDataset.bind(this); + + this.getExemplarDatasets = this.getExemplarDatasets.bind(this); + this.getExemplarDataset = this.getExemplarDataset.bind(this); } public async getDatasets(req: Request, res: Response): Promise { try { - const { q = '', offset = '0', limit = '0'}: { q: string; offset: string; limit: string } = req.query; + const { q = '', offset = '0', limit = '0' }: { q: string; offset: string; limit: string } = req.query; const limitInt = parseInt(limit); const offsetInt = parseInt(offset); const datasets = await this.datasetService.getDatasets(q, offsetInt, limitInt); const datasetTotal = await this.datasetService.getDatasetCount(); - return res - .status(200) - .json(datasets); + return res.status(200).json(datasets); } catch (err) { this._logger.sendDataInLogging({ data: err.name }, 'ERROR'); this._logger.sendDataInLogging({ data: (err).message }, 'ERROR'); @@ -49,4 +50,37 @@ export default class DatasetController extends BaseController { return res.status(500).json({ status: 'error', message: (err).message }); } } + + public async getExemplarDatasets(req: Request, res: Response): Promise { + try { + const { q = '', offset = '0', limit = '0' }: { q: string; offset: string; limit: string } = req.query; + + const limitInt = parseInt(limit); + const offsetInt = parseInt(offset); + const datasets = await this.datasetService.getExemplarDatasets(q, offsetInt, limitInt); + const datasetTotal = await this.datasetService.getExemplarDatasetCount(); + + return res.status(200).json(datasets); + } catch (err) { + this._logger.sendDataInLogging({ data: err.name }, 'ERROR'); + this._logger.sendDataInLogging({ data: (err).message }, 'ERROR'); + return res.status(500).json({ status: 'error', message: (err).message }); + } + } + + public async getExemplarDataset(req: Request, res: Response): Promise { + try { + const pid: string = req.params.persistentId; + const dataset: any = await this.datasetService.getExemplarDataset(pid); + + if (!dataset) { + return res.status(404).json({ status: 'error', message: 'Dataset not found' }); + } + + return res.status(200).json(dataset); + } catch (err) { + this._logger.sendDataInLogging({ data: (err).message }, 'ERROR'); + return res.status(500).json({ status: 'error', message: (err).message }); + } + } } diff --git a/src/api/v1/routes/dataset.route.ts b/src/api/v1/routes/dataset.route.ts index 49b2b1c..f5c0564 100644 --- a/src/api/v1/routes/dataset.route.ts +++ b/src/api/v1/routes/dataset.route.ts @@ -13,6 +13,9 @@ const datasetController = new DatasetController(datasetService); router.get('/noauth/datasets', validateParams, datasetController.getDatasets); router.get('/noauth/datasets/:persistentId', datasetController.getDataset); +router.get('/noauth/exemplar/datasets', validateParams, datasetController.getExemplarDatasets); +router.get('/noauth/exemplar/datasets/:persistentId', datasetController.getExemplarDataset); + // Protected endpoints - API key or OAuth access token router.get('/datasets', authorise('user'), validateParams, datasetController.getDatasets); router.get('/datasets/:persistentId', authorise('user'), datasetController.getDataset); diff --git a/src/api/v1/services/dataset.service.ts b/src/api/v1/services/dataset.service.ts index e30f883..fdbf582 100644 --- a/src/api/v1/services/dataset.service.ts +++ b/src/api/v1/services/dataset.service.ts @@ -5,20 +5,40 @@ export default class DatasetService { public async getDatasets(q: string, offset: number, limit: number): Promise { let datasets = []; - const file = fs.readFileSync("./files/list.json", 'utf-8'); + const file = fs.readFileSync('./files/list.json', 'utf-8'); const parsedJSON = JSON.parse(file); return parsedJSON; } public async getDataset(pid: string) { - const file = fs.readFileSync("./files/dataset.json", 'utf-8'); + const file = fs.readFileSync('./files/dataset.json', 'utf-8'); const parsedJSON = JSON.parse(file); return parsedJSON; } public async getDatasetCount() { - const files = fs.readdirSync("./files"); + const files = fs.readdirSync('./files'); + return files.length; + } + + public async getExemplarDatasets(q: string, offset: number, limit: number): Promise { + let datasets = []; + + const file = fs.readFileSync('./files/exemplar/list.json', 'utf-8'); + const parsedJSON = JSON.parse(file); + return parsedJSON; + } + + public async getExemplarDataset(pid: string) { + const file = fs.readFileSync(`./files/exemplar/${pid}.json`, 'utf-8'); + const parsedJSON = JSON.parse(file); + + return parsedJSON; + } + + public async getExemplarDatasetCount() { + const files = fs.readdirSync('./files'); return files.length; } } diff --git a/src/server.ts b/src/server.ts index f8f3e4f..281820d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -36,7 +36,7 @@ const initApplication = async () => { res.status(404).end('404 - not found'); }); - await Database.init(); + //await Database.init(); app.listen(port, () => { process.stdout.write(`Service running @ 'http://localhost:${port}'\n`); From 76d7aaaea9aeaba07d1fed4ffcfd71f100a7fa2e Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Fri, 16 Feb 2024 10:15:38 +0000 Subject: [PATCH 2/8] turn off some db crap --- src/server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 281820d..0db7b94 100644 --- a/src/server.ts +++ b/src/server.ts @@ -36,7 +36,9 @@ const initApplication = async () => { res.status(404).end('404 - not found'); }); - //await Database.init(); + if (Locals.config().MONGO_DATABASE != '') { + await Database.init(); + } app.listen(port, () => { process.stdout.write(`Service running @ 'http://localhost:${port}'\n`); From a063c798896901faf5353e735007ec5f6bfdb396 Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Fri, 16 Feb 2024 10:47:30 +0000 Subject: [PATCH 3/8] trigger build --- .github/workflows/dev_deployment.yaml | 167 +++++++++++++------------- 1 file changed, 85 insertions(+), 82 deletions(-) diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml index dbe000a..0481ca4 100644 --- a/.github/workflows/dev_deployment.yaml +++ b/.github/workflows/dev_deployment.yaml @@ -2,99 +2,102 @@ name: Build and deploy to DEV run-name: ${{ github.actor }} triggered deploy to DEV pipeline on: - push: - branches: - - 'main' + push: + branches: + #- 'main' + - 'add/exemplar-datasets' env: - PROJECT_ID: '${{ secrets.PROJECT_ID }}' - GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' + PROJECT_ID: '${{ secrets.PROJECT_ID }}' + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' jobs: - build: - permissions: - contents: write - id-token: write + build: + permissions: + contents: write + id-token: write - runs-on: ubuntu-latest - environment: dev - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: main + runs-on: ubuntu-latest + environment: dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: add/exemplar-datasets + #ref: main - - name: Read VERSION file - id: getversion - run: | - sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version - echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT + - name: Read VERSION file + id: getversion + run: | + sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version + echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - - name: Login to GAR - id: garlogin - uses: docker/login-action@v2 - with: - registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }} - username: oauth2accesstoken - password: ${{ steps.auth.outputs.access_token }} + - name: Login to GAR + id: garlogin + uses: docker/login-action@v2 + with: + registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }} + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} - - name: Build and Push Container - id: build - shell: bash - env: - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - PROJECT_ID: ${{ secrets.PROJECT_ID }} - GAR_NAME: ${{ secrets.GAR_NAME }} - - run: |- - docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./ - docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }} + - name: Build and Push Container + id: build + shell: bash + env: + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_NAME: ${{ secrets.GAR_NAME }} - deploy: - needs: build - permissions: - contents: write - id-token: write + run: |- + docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./ + docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }} - runs-on: ubuntu-latest - environment: dev - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: main + deploy: + needs: build + permissions: + contents: write + id-token: write - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + runs-on: ubuntu-latest + environment: dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: add/exemplar-datasets + #ref: main - - name: Read VERSION file - id: getversion - run: | - sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version - echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT - - - name: Deploy to Cloud Run - uses: actions-hub/gcloud@master - id: deploy - env: - PROJECT_ID: ${{ secrets.PROJECT_ID }} - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - GAR_NAME: ${{ secrets.GAR_NAME }} - SERVICE_NAME: '${{ secrets.SERVICE_NAME }}' - SERVICE_REGION: '${{ secrets.SERVICE_REGION }}' + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - with: - args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' \ No newline at end of file + - name: Read VERSION file + id: getversion + run: | + sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version + echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION }}' + + with: + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' From be2ab4a557760647e2d1836dbf9cef066620a602 Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Fri, 16 Feb 2024 12:31:05 +0000 Subject: [PATCH 4/8] revert ci --- .github/workflows/dev_deployment.yaml | 167 +++++++++++++------------- 1 file changed, 82 insertions(+), 85 deletions(-) diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml index 0481ca4..dbe000a 100644 --- a/.github/workflows/dev_deployment.yaml +++ b/.github/workflows/dev_deployment.yaml @@ -2,102 +2,99 @@ name: Build and deploy to DEV run-name: ${{ github.actor }} triggered deploy to DEV pipeline on: - push: - branches: - #- 'main' - - 'add/exemplar-datasets' + push: + branches: + - 'main' env: - PROJECT_ID: '${{ secrets.PROJECT_ID }}' - GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' + PROJECT_ID: '${{ secrets.PROJECT_ID }}' + GAR_LOCATION: '${{ secrets.GAR_LOCATION }}' jobs: - build: - permissions: - contents: write - id-token: write + build: + permissions: + contents: write + id-token: write - runs-on: ubuntu-latest - environment: dev - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: add/exemplar-datasets - #ref: main + runs-on: ubuntu-latest + environment: dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main - - name: Read VERSION file - id: getversion - run: | - sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version - echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT + - name: Read VERSION file + id: getversion + run: | + sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version + echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - - name: Login to GAR - id: garlogin - uses: docker/login-action@v2 - with: - registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }} - username: oauth2accesstoken - password: ${{ steps.auth.outputs.access_token }} + - name: Login to GAR + id: garlogin + uses: docker/login-action@v2 + with: + registry: ${{ env.GAR_LOCATION }}-docker.pkg.dev/${{ env.PROJECT_ID }}/${{ env.PROJECT_ID }} + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} - - name: Build and Push Container - id: build - shell: bash - env: - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - PROJECT_ID: ${{ secrets.PROJECT_ID }} - GAR_NAME: ${{ secrets.GAR_NAME }} + - name: Build and Push Container + id: build + shell: bash + env: + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_NAME: ${{ secrets.GAR_NAME }} + + run: |- + docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./ + docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }} - run: |- - docker build -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} -t '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:latest ./ - docker push --all-tags '${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }} + deploy: + needs: build + permissions: + contents: write + id-token: write - deploy: - needs: build - permissions: - contents: write - id-token: write + runs-on: ubuntu-latest + environment: dev + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: main - runs-on: ubuntu-latest - environment: dev - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: add/exemplar-datasets - #ref: main + - name: Google Auth + id: auth + uses: 'google-github-actions/auth@v0' + with: + token_format: 'access_token' + workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' + service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' - - name: Google Auth - id: auth - uses: 'google-github-actions/auth@v0' - with: - token_format: 'access_token' - workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' - service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' + - name: Read VERSION file + id: getversion + run: | + sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version + echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT + + - name: Deploy to Cloud Run + uses: actions-hub/gcloud@master + id: deploy + env: + PROJECT_ID: ${{ secrets.PROJECT_ID }} + GAR_LOCATION: ${{ secrets.GAR_LOCATION }} + GAR_NAME: ${{ secrets.GAR_NAME }} + SERVICE_NAME: '${{ secrets.SERVICE_NAME }}' + SERVICE_REGION: '${{ secrets.SERVICE_REGION }}' - - name: Read VERSION file - id: getversion - run: | - sed -n 's/^appVersion:\(.*\)/\1/p' < chart/fma/Chart.yaml > version - echo "version=$(sed '/.*\"\(.*\)\".*/ s//\1/g' version)" >> $GITHUB_OUTPUT - - - name: Deploy to Cloud Run - uses: actions-hub/gcloud@master - id: deploy - env: - PROJECT_ID: ${{ secrets.PROJECT_ID }} - GAR_LOCATION: ${{ secrets.GAR_LOCATION }} - GAR_NAME: ${{ secrets.GAR_NAME }} - SERVICE_NAME: '${{ secrets.SERVICE_NAME }}' - SERVICE_REGION: '${{ secrets.SERVICE_REGION }}' - - with: - args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' + with: + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' \ No newline at end of file From d9cc5ce52418d0e170c568fb0534286f70dfed90 Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Fri, 16 Feb 2024 12:32:26 +0000 Subject: [PATCH 5/8] fix for schema diff --- files/exemplar/list.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/exemplar/list.json b/files/exemplar/list.json index 40889be..b5d45f1 100644 --- a/files/exemplar/list.json +++ b/files/exemplar/list.json @@ -26,7 +26,7 @@ }, { "name": "UKBiobank with tissues", - "@schema": "https://raw.githubusercontent.com/HDRUK/schemata-2/master/hdr_schemata/models/HDRUK/2.1.2/schema.json", + "@schema": "https://raw.githubusercontent.com/HDRUK/schemata-2/master/hdr_schemata/models/HDRUK/2.2.0/schema.json", "description": "Examplar UKBiobank", "type": "dataset", "persistentId": "1460e2b4-a985-4890-8e60-a21e78ce01f3", From a92b3108af49d78eca5f99919f27cbb85fefe25d Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Mon, 19 Feb 2024 11:16:18 +0000 Subject: [PATCH 6/8] rename file --- ...aebb4c.json.json => 7eb5d9ff-0d54-4438-9c60-423848aebb4c.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename files/exemplar/{7eb5d9ff-0d54-4438-9c60-423848aebb4c.json.json => 7eb5d9ff-0d54-4438-9c60-423848aebb4c.json} (100%) diff --git a/files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json.json b/files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json similarity index 100% rename from files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json.json rename to files/exemplar/7eb5d9ff-0d54-4438-9c60-423848aebb4c.json From f5e66918d8d27a87cbc18bb90bd61670f7b4fa59 Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Mon, 19 Feb 2024 11:18:06 +0000 Subject: [PATCH 7/8] revert --- .github/workflows/dev_deployment.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml index dbe000a..eb094e0 100644 --- a/.github/workflows/dev_deployment.yaml +++ b/.github/workflows/dev_deployment.yaml @@ -4,7 +4,7 @@ run-name: ${{ github.actor }} triggered deploy to DEV pipeline on: push: branches: - - 'main' + - 'add/exemplar-datasets' env: PROJECT_ID: '${{ secrets.PROJECT_ID }}' @@ -22,7 +22,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - ref: main + ref: add/exemplar-datasets - name: Read VERSION file id: getversion @@ -70,7 +70,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - ref: main + ref: add/exemplar-datasets - name: Google Auth id: auth @@ -97,4 +97,4 @@ jobs: SERVICE_REGION: '${{ secrets.SERVICE_REGION }}' with: - args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' \ No newline at end of file + args: run services update '${{ env.SERVICE_NAME }}' --image='${{ env.GAR_LOCATION }}'-docker.pkg.dev/'${{ env.PROJECT_ID }}'/'${{ env.GAR_NAME }}'/${{ steps.getversion.outputs.version }}:${{ github.sha }} --region='${{ env.SERVICE_REGION }}' --project='${{ env.PROJECT_ID }}' From f99468b755a567b534eca1afd6d82236560b0a1b Mon Sep 17 00:00:00 2001 From: Calum Macdonald Date: Mon, 19 Feb 2024 11:18:19 +0000 Subject: [PATCH 8/8] turn off again --- .github/workflows/dev_deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev_deployment.yaml b/.github/workflows/dev_deployment.yaml index eb094e0..f802487 100644 --- a/.github/workflows/dev_deployment.yaml +++ b/.github/workflows/dev_deployment.yaml @@ -4,7 +4,7 @@ run-name: ${{ github.actor }} triggered deploy to DEV pipeline on: push: branches: - - 'add/exemplar-datasets' + - 'main' env: PROJECT_ID: '${{ secrets.PROJECT_ID }}' @@ -22,7 +22,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - ref: add/exemplar-datasets + ref: main - name: Read VERSION file id: getversion @@ -70,7 +70,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 with: - ref: add/exemplar-datasets + ref: main - name: Google Auth id: auth