Merge remote-tracking branch 'upstream/dev' into dev

package.json

@@ -43,6 +43,7 @@
     "csv-parse": "^5.4.1",
     "csv-parser": "^3.0.0",
     "csvtojson": "^2.0.10",
+    "decompress": "^4.2.1",
     "express": "^4.18.2",
     "fast-csv": "^4.3.6",
     "form-data": "^4.0.0",

src/ingestion/controller/ingestion.controller.ts

@@ -10,15 +10,17 @@ import {
     Result, EmissionBody, RawDataPullBody
 } from '../interfaces/Ingestion-data';
 import {
+    BadRequestException,
     Body,
     Controller, FileTypeValidator,
     Get,
-    MaxFileSizeValidator,
+    Logger,
     ParseFilePipe,
     Post,
     Query,
     Res,
     UploadedFile,
+    UploadedFiles,
     UseInterceptors,
     Put,
     UseGuards, Req, Param
@@ -29,7 +31,6 @@ import {RawDataImportService} from '../services/rawDataImport/rawDataImport.serv
 import {EventService} from '../services/event/event.service';
 import {Response, Request} from 'express';
 import {CsvImportService} from "../services/csvImport/csvImport.service";
-import {FileInterceptor} from "@nestjs/platform-express";
 import {diskStorage} from "multer";
 import {FileIsDefinedValidator} from "../validators/file-is-defined-validator";
 import {FileStatusService} from '../services/file-status/file-status.service';
@@ -43,6 +44,13 @@ import { NvskApiService } from '../services/nvsk-api/nvsk-api.service';
 import { UploadDimensionFileService } from '../services/upload-dimension-file/upload-dimension-file.service';
 import { GrammarService } from '../services/grammar/grammar.service';
 import { GenericFunction } from '../services/generic-function';
+import {
+    FileFieldsInterceptor,
+    FileInterceptor,
+} from '@nestjs/platform-express';
+import { FileType, FileValidateRequest } from '../dto/request';
+import * as fs from 'fs';
+import { ValidatorService } from '../services/validator/validator.service';
 
 let validateBodySchema = {
     "type": "object",
@@ -65,9 +73,18 @@ let validateBodySchema = {
     ]
 };
 
+const defaultStorageConfig = diskStorage({
+    destination: './upload',
+    filename: (req, file, cb) => {
+      cb(null, Date.now() + '-' + file.originalname);
+    },
+});
+
 @ApiTags('ingestion')
 @Controller('')
 export class IngestionController {
+    private logger: Logger;
+
     constructor(
         private datasetService: DatasetService, private dimensionService: DimensionService
         , private eventService: EventService, private csvImportService: CsvImportService, private fileStatus: FileStatusService, private updateFileStatus: UpdateFileStatusService,
@@ -76,7 +93,9 @@ export class IngestionController {
         private nvskService:NvskApiService,
         private grammarService: GrammarService,
         private service: GenericFunction,
-        private uploadDimension:UploadDimensionFileService) {
+        private uploadDimension:UploadDimensionFileService,
+        private validatorService: ValidatorService) {
+        this.logger = new Logger(IngestionController.name);
     }
 
     @Get('generatejwt')
@@ -327,7 +346,7 @@ export class IngestionController {
         })
     }))
 
-    @Post('/validate')
+    @Post('/validate-old')
     @ApiConsumes('multipart/form-data')
     async validateEventOrDimension(@Body() body: any, @Res()response: Response, @UploadedFile(
         new ParseFilePipe({
@@ -357,4 +376,100 @@ export class IngestionController {
             // throw new Error(e);
         }
     }
+
+    @Post('validate')
+    @UseInterceptors(
+        FileFieldsInterceptor(
+            [
+                { name: 'grammar', maxCount: 1 },
+                { name: 'data', maxCount: 1 },
+            ],
+            {
+                storage: defaultStorageConfig,
+                fileFilter(req, file, callback) {
+                if (file.mimetype !== 'text/csv') {
+                    return callback(
+                    new BadRequestException('Only CSV files are allowed'),
+                    false,
+                    );
+                }
+                callback(null, true);
+                },
+            },
+        )
+    )
+    uploadFileN(
+        @UploadedFiles()
+        files: {
+        grammar?: Express.Multer.File[];
+        data?: Express.Multer.File[];
+        },
+        @Body() body: FileValidateRequest,
+    ) {
+        this.logger.debug(files.grammar);
+        const grammarFilePath = files.grammar[0].path;
+
+        if (!grammarFilePath || !fs.existsSync(grammarFilePath))
+        throw new BadRequestException('Grammar file is required');
+
+        const grammarContent = fs.readFileSync(grammarFilePath, 'utf8');
+        const dataFilePath = files?.data ? files?.data[0]?.path : undefined;
+
+        let resp;
+        switch (body.type.trim()) {
+        case FileType.DimensionGrammar:
+            resp =
+            this.validatorService.checkDimensionGrammarForValidationErrors(
+                grammarContent,
+            );
+            break;
+        case FileType.DimensionData:
+            if (!dataFilePath || !fs.existsSync(dataFilePath))
+            throw new BadRequestException('Data file is required');
+
+            resp = this.validatorService.checkDimensionDataForValidationErrors(
+            grammarContent,
+            fs.readFileSync(dataFilePath, 'utf8'),
+            );
+            break;
+        case FileType.EventGrammar:
+            resp =
+            this.validatorService.checkEventGrammarForValidationErrors(
+                grammarContent,
+            );
+            break;
+        case FileType.EventData:
+            if (!dataFilePath || !fs.existsSync(dataFilePath))
+            throw new BadRequestException('Data file is required');
+
+            resp = this.validatorService.checkEventDataForValidationErrors(
+            grammarContent,
+            fs.readFileSync(dataFilePath, 'utf8'),
+            );
+            break;
+        default:
+            throw new BadRequestException('Invalid file type');
+        }
+
+        // delete the files
+        if (grammarFilePath) fs.unlinkSync(grammarFilePath);
+        if (dataFilePath) fs.unlinkSync(dataFilePath);
+
+        return resp;
+    }
+
+    @Post('bulk')
+    @UseInterceptors(
+        FileInterceptor('folder', {
+        storage: defaultStorageConfig,
+        }),
+    )
+    async uploadBulkZip(@UploadedFile() file: Express.Multer.File) {
+        const zipFilePath = file.path;
+
+        const resp = await this.validatorService.handleZipFile(zipFilePath);
+        // delete the file
+        if (zipFilePath) fs.unlinkSync(zipFilePath);
+        return resp;
+    }
 }

src/ingestion/cqube-spec-checker/dimension.data.validator.ts

@@ -0,0 +1,54 @@
+export class DimensionDataValidator {
+  grammarContent: any;
+  lines: any;
+  pkIndexLine: any;
+  dataTypesLine: any;
+  headerLine: any;
+  dataContent: any;
+  dataContentLines: any;
+  errors: any[];
+  constructor(grammarContent, dataContent) {
+    this.grammarContent = grammarContent;
+    this.lines = this.grammarContent.trim().split('\n');
+    this.pkIndexLine = this.lines[0].trim().split(',');
+    this.dataTypesLine = this.lines[1].trim().split(',');
+    this.headerLine = this.lines[2].trim().split(',');
+    this.dataContent = dataContent;
+    this.dataContentLines = this.dataContent
+      .trim()
+      .split('\n')[0]
+      .trim()
+      .split(',');
+    this.errors = [];
+  }
+
+  verify() {
+    this.verifyColumnsToGrammar();
+    return this.errors;
+  }
+
+  verifyColumnsToGrammar() {
+    this.headerLine.forEach((header, index) => {
+      this.dataContentLines.indexOf(header) === -1
+        ? this.errors.push({
+          row: 0,
+          col: index,
+          errorCode: 1001,
+          error: `Missing header from grammar file: ${header}`,
+        })
+        : null;
+    });
+
+    this.dataContentLines.forEach((header, index) => {
+      this.headerLine.indexOf(header) === -1
+        ? this.errors.push({
+          row: 0,
+          col: index,
+          errorCode: 1001,
+          error: `Extra header not in grammar file: ${header}`,
+          data: header,
+        })
+        : null;
+    });
+  }
+}

src/ingestion/cqube-spec-checker/dimension.grammar.validator.ts

@@ -0,0 +1,80 @@
+export class DimensionValidator {
+  content: any;
+  lines: any;
+  pkIndexLine: any;
+  dataTypesLine: any;
+  headerLine: any;
+  constructor(content) {
+    this.content = content;
+    this.lines = this.content.trim().split('\n');
+    this.pkIndexLine = this.lines[0].trim().split(',');
+    this.dataTypesLine = this.lines[1].trim().split(',');
+    this.headerLine = this.lines[2].trim().split(',');
+  }
+
+  verify() {
+    const errors = [];
+    errors.push(...this.verifyColumns());
+    errors.push(...this.verifyPkIndexLine());
+    errors.push(...this.verifyDataTypes());
+    return errors;
+  }
+
+  verifyColumns() {
+    const errors = [];
+    const columnCount = this.pkIndexLine.length;
+    this.lines.forEach((line, lineNumber) => {
+      if (line !== '') {
+        // Ignore last line
+        const lineColumns = line.split(',').length;
+        if (lineColumns !== columnCount) {
+          errors.push({
+            row: lineNumber,
+            col: 0,
+            errorCode: 2003,
+            error: `Line ${lineNumber + 1
+              }: Invalid number of columns ${lineColumns} (expected ${columnCount}), ${line.split(
+                ',',
+              )}`,
+            data: line,
+          });
+        }
+      }
+    });
+    return errors;
+  }
+
+  verifyPkIndexLine() {
+    const errors = [];
+    if (
+      this.pkIndexLine.indexOf('PK') === -1 ||
+      this.pkIndexLine.indexOf('Index') === -1
+    ) {
+      errors.push({
+        row: 0,
+        col: 0,
+        errorCode: 1003,
+        error: `Invalid PK/Index: First row must include 'PK' and 'Index' but found "${this.pkIndexLine}"`,
+        data: this.pkIndexLine,
+      });
+    }
+    return errors;
+  }
+
+  verifyDataTypes() {
+    const errors = [];
+    this.dataTypesLine.forEach((dataType, columnIndex) => {
+      if (dataType !== 'string' && dataType !== 'integer') {
+        errors.push({
+          row: 1,
+          col: columnIndex,
+          errorCode: 1002,
+          error: `Invalid data type at column ${columnIndex + 1
+            }: Only 'string' and 'integer' are allowed but found '${dataType}'`,
+          data: this.dataTypesLine,
+        });
+      }
+    });
+    return errors;
+  }
+}

src/ingestion/dto/errors.ts

@@ -0,0 +1,7 @@
+export type ValidationErrors = {
+  row: string | number;
+  col: string | number;
+  errorCode: number;
+  error: string;
+  data?: any;
+};

src/ingestion/dto/request.ts

@@ -0,0 +1,10 @@
+export enum FileType {
+  DimensionGrammar = 'dimension-grammar',
+  DimensionData = 'dimension-data',
+  EventGrammar = 'event-grammar',
+  EventData = 'event-data',
+}
+
+export class FileValidateRequest {
+  type: FileType;
+}

src/ingestion/dto/response.ts

@@ -0,0 +1,5 @@
+import { ValidationErrors } from './errors';
+
+export class SingleFileValidationResponse {
+  errors: ValidationErrors[];
+}

src/ingestion/ingestion.module.ts

@@ -17,11 +17,12 @@ import { NvskApiService } from './services/nvsk-api/nvsk-api.service';
 import { DateService } from './services/dateService';
 import { UploadDimensionFileService } from './services/upload-dimension-file/upload-dimension-file.service';
 import { GrammarService } from './services/grammar/grammar.service';
+import { ValidatorService } from './services/validator/validator.service';
 
 @Module({
     controllers: [IngestionController],
     providers: [DatasetService, DimensionService, EventService, GenericFunction, HttpCustomService, CsvImportService, FileStatusService,
-        UpdateFileStatusService, DataEmissionService, UploadDimensionFileService, UploadService,RawDataImportService,NvskApiService,DateService,GrammarService],
+        UpdateFileStatusService, DataEmissionService, UploadDimensionFileService, UploadService,RawDataImportService,NvskApiService,DateService,GrammarService, ValidatorService],
     imports: [DatabaseModule, HttpModule]
 })
 export class IngestionModule {

src/ingestion/services/grammar/grammar.service.ts

@@ -7,11 +7,11 @@ export class GrammarService {
     }
 
     async getEventSchemas() {
-        return await this._databaseService.executeQuery(`select id, name, schema from spec."EventGrammar" WHERE eventType='EXTERNAL'`);
+        return await this._databaseService.executeQuery(`select id, name, schema from spec."EventGrammar" WHERE "eventType"='EXTERNAL'`);
     }
 
     async getDimensionSchemas() {
-        return await this._databaseService.executeQuery(`select id, name, schema from spec."DimensionGrammar" WHERE dimensionType='EXTERNAL'`);
+        return await this._databaseService.executeQuery(`select id, name, schema from spec."DimensionGrammar" WHERE "dimensionType"='EXTERNAL'`);
     }
 
     async getEventSchemaByID(id) {