Merge pull request #2 from HDRUK/feature/GAT-3129

Feature/gat 3129 Validation of HDR schema for MK1 simulation server for FMA

.env.example

@@ -0,0 +1,12 @@
+PORT=5000
+
+MONGO_USER="" # Username with RW access to the relevant database
+MONGO_PASSWORD=""
+MONGO_CLUSTER="" # Relevant cluster
+MONGO_DATABASE="" # Name of relevant database on cluster
+
+JWT_SECRET=""
+
+LOGGING_TYPE= "local" # Whether to log locally ("local") or to GCP
+LOGGING_PROJECT_ID="<GCP project ID>"
+LOGGING_LOG_NAME="<GCP logs name>"

files/2.json

@@ -0,0 +1 @@
+{"@schema" : "https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/2.1.0/dataset.schema.json", "identifier": "e96e36ba-30ca-4c25-bc55-fab02d72a51c", "version": "1.0.0", "issued": "2021-08-30T21:00:00+00:00", "modified": "2021-08-30T21:00:00+00:00", "revisions": [], "summary": {"title": "Bones Dataset", "abstract": "Test description", "publisher": {"identifier": "http://bones.com", "name": "Bones", "logo": "http://example.com", "description": "A publisher", "contactPoint": [], "memberOf": "ALLIANCE", "accessRights": [], "accessService": "Many many options", "accessRequestCost": "Thousands of pounds"}, "contactPoint": "[email protected]", "keywords": ["bones", "Blood"], "doiName": "10.1093/ajae/aaq063"}, "coverage": {"spatial": [], "typicalAgeRange": "25-80"}, "provenance": {"origin": {"purpose": "COVID-19", "source": "Routine Surveilance"}, "temporal": {"accrualPeriodicity": "IRREGULAR", "distributionReleaseDate": "2021-08-30T21:00:00+00:00", "startDate": "2021-08-10T21:00:00+00:00", "endDate": "2021-08-30T21:00:00+00:00", "timeLag": "1-2 WEEKS"}}, "accessibility": {"usage": { "dataUseRequirements": "none", "resourceCreator": "Someone", "investigations": [], "isReferencedBy": []}, "access": {"accessRights": ["http://test.bones.com"], "accessService": "Many many options", "accessRequestCost": "Thousands", "jurisdiction": ["GB-NIR"], "dataProcessor": "Some guy somewhere", "dataController": "This is the data controller"}, "formatAndStandards": {"vocabularyEncodingScheme": ["OPCS4"], "conformsTo": ["HL7 CDA"], "language": ["en"], "format": ["audio"]}}, "enrichmentAndLinkage": {"qualifiedRelation": [], "derivation": [], "tools": []}, "observations": [{"observedNode": "PERSONS", "measuredValue": 3, "observationDate": "2021-08-10T21:00:00+00:00", "measuredProperty": "Count"}]}

package.json

@@ -40,7 +40,7 @@
         "sinon": "^13.0.1",
         "ts-jest": "^27.1.3",
         "ts-node": "^10.5.0",
-        "typescript": "^4.5.5"
+        "typescript": "^4.9.5"
     },
     "scripts": {
         "start": "tsc && node dist/server.js",

src/api/v1/config/locals.ts

@@ -10,8 +10,9 @@ class Locals {
         const JWTSecret = process.env.JWT_SECRET;
         const googleProjectId = process.env.LOGGING_PROJECT_ID;
         const googleLoggingName = process.env.LOGGING_LOG_NAME;
+        const loggingType = process.env.LOGGING_TYPE;
 
-        return { mongoURI, port, JWTSecret, googleProjectId, googleLoggingName };
+        return { mongoURI, port, JWTSecret, googleProjectId, googleLoggingName, loggingType };
     }
 }
 

src/api/v1/controllers/dataset.controller.ts

@@ -17,15 +17,18 @@ export default class DatasetController extends BaseController {
 
     public async getDatasets(req: Request, res: Response): Promise<any> {
         try {
-            const { q = '', offset = '0', limit }: { q: string; offset: string; limit: string } = <any>req.query;
+            const { q = '', offset = '0', limit = '0'}: { q: string; offset: string; limit: string } = <any>req.query;
 
-            const datasets = await this.datasetService.getDatasets(q, parseInt(offset), parseInt(limit));
+            const limitInt = parseInt(limit);
+            const offsetInt = parseInt(offset);
+            const datasets = await this.datasetService.getDatasets(q, offsetInt, limitInt);
             const datasetTotal = await this.datasetService.getDatasetCount();
 
             return res
                 .status(200)
-                .json({ query: { q: q || '', total: String(datasetTotal), limit: limit || '', offset: offset }, items: datasets });
+                .json({ query: { q: q || '', total: datasetTotal, limit: limitInt, offset: offsetInt }, items: datasets });
         } catch (err) {
+            this._logger.sendDataInLogging({ data: err.name }, 'ERROR');
             this._logger.sendDataInLogging({ data: (<Error>err).message }, 'ERROR');
             return res.status(500).json({ status: 'error', message: (<Error>err).message });
         }

src/api/v1/services/dataset.service.ts

@@ -1,53 +1,91 @@
-import { Datasets } from '../models/dataset.model';
-
 import { dataset } from '../types/dataset.interface';
+import fs from 'fs';
 
 export default class DatasetService {
     public async getDatasets(q: string, offset: number, limit: number): Promise<dataset[]> {
-        const pipeline: Array<any> = [
+        // The files directory contains a set of example JSON files
+        const files = fs.readdirSync("./files");
+        let datasets = [];
+
+        for (let i = 0; i < files.length; i++)
+        {
+            const file = fs.readFileSync("./files/" + files[i], 'utf-8');
+            const parsedJSON = JSON.parse(file);
+            const datasetData = {
+                persistentId : parsedJSON.identifier,
+                "@schema" : parsedJSON["@schema"],
+                version : parsedJSON.version,
+                issued : parsedJSON.issued,
+                modified : parsedJSON.modified,
+                name : parsedJSON.summary.title,
+                description : parsedJSON.summary.abstract,
+                type : "dataset",
+                source : parsedJSON.summary.publisher.name,
+                self : "http://example-url.com/api/datasets/" + parsedJSON.identifier
+            }
+
+            // If the search query isn't blank
+            if (q !== '')
+            {
+                // If the query can be found in the description
+                if (parsedJSON.summary.abstract.search(q) !== -1)
+                {
+                    datasets.push(datasetData);
+                }
+            }
+            else
             {
-                $match: {
-                    ...(q && { $text: { $search: q } }),
-                    status: 'active',
-                },
-            },
-            { $skip: offset },
-        ];
-
-        if (limit) pipeline.push({ $limit: limit });
-
-        const datasets = await Datasets.aggregate(pipeline).exec();
-
-        const mappedDatasets: dataset[] = await datasets.map((dataset) => {
-            return {
-                '@schema': dataset.schema,
-                type: 'dataset',
-                persistentId: dataset.persistentId,
-                name: dataset.datasetv2.summary.title,
-                description: dataset.datasetv2.documentation.description,
-                version: dataset.datasetv2.version,
-                issued: dataset.datasetv2.issued,
-                modified: dataset.datasetv2.modified,
-                source: dataset.datasetv2.summary.publisher.name,
-            };
-        });
-
-        return mappedDatasets;
+                datasets.push(datasetData);
+            }
+        }
+
+        // Check the offset is valid
+        if (offset < datasets.length)
+        {
+            datasets = datasets.slice(offset);
+        }
+
+        if (!isNaN(limit) && limit > 0)
+        {
+            datasets = datasets.slice(0, limit);
+        }
+
+        return datasets;
     }
 
     public async getDataset(pid: string) {
-        const datasets = await Datasets.find({ persistentId: pid }).sort({ 'datasetv2.version': -1 }).select('datasetv2').lean();
 
-        if (datasets.length > 0) {
-            return datasets[0].datasetv2;
-        }
+        // The files directory contains a set of example JSON files
+        const files = fs.readdirSync("./files");
+
+        for (let i = 0; i < files.length; i++)
+        {
+            const file = fs.readFileSync("./files/" + files[i], 'utf-8');
+            const parsedJSON = JSON.parse(file);
 
+            // This is the dataset the user is looking for
+            if (parsedJSON.identifier === pid)
+            {
+                return {
+                    persistentId : parsedJSON.identifier,
+                    "@schema" : parsedJSON["@schema"],
+                    version : parsedJSON.version,
+                    issued : parsedJSON.issued,
+                    modified : parsedJSON.modified,
+                    name : parsedJSON.summary.title,
+                    description : parsedJSON.summary.abstract,
+                    type : "dataset",
+                    source : parsedJSON.summary.publisher.name,
+                    self : "http://example-url.com/api/datasets/" + parsedJSON.identifier
+                }
+            }
+
+        }
         return;
     }
 
     public async getDatasetCount() {
-        const count = await Datasets.count({ status: 'active' });
-
-        return count;
+        const files = fs.readdirSync("./files");
+        return files.length;
     }
 }

src/api/v1/services/google/logging.service.ts

@@ -5,15 +5,29 @@ import Locals from '../../config/locals';
 export default class LoggingService {
     _logger;
     constructor() {
+        if (Locals.config().loggingType !== 'gcp') {
+            return;
+        }
         this._logger = LoggerGCP.createLoggerGCP({
             projectId: Locals.config().googleProjectId,
             logName: Locals.config().googleLoggingName,
         });
     }
 
     sendDataInLogging(data: any, severity: any) {
-        this._logger.setData(data);
-        this._logger.setSeverity(severity);
-        this._logger.writeLog();
+
+        if (Locals.config().loggingType !== "gcp")
+        {
+            process.stdout.write(severity.toString());
+            process.stdout.write(" - ");
+            process.stdout.write(data.toString());
+            process.stdout.write("\n");
+        }
+        else
+        {
+            this._logger.setData(data);
+            this._logger.setSeverity(severity);
+            this._logger.writeLog();
+        }
     }
 }