From ef6f60bd257e8e09e2e5243881527ba3f32e710f Mon Sep 17 00:00:00 2001 From: Yash Mittal Date: Mon, 30 Oct 2023 18:28:40 +0530 Subject: [PATCH] feat: remove dependency on grammar files during `ingest-data` --- .../migration.sql | 8 + .../migration.sql | 8 + .../migration.sql | 2 + impl/c-qube/prisma/schema.prisma | 2 + .../csv-adapter/csv-adapter.service.ts | 143 +++++++++--------- .../event-grammar/event-grammar.service.ts | 23 +++ .../src/services/event/event.service.ts | 10 ++ .../query-builder/query-builder.service.ts | 2 +- 8 files changed, 126 insertions(+), 72 deletions(-) create mode 100644 impl/c-qube/prisma/migrations/20231030113226_add_metric_in_eg/migration.sql create mode 100644 impl/c-qube/prisma/migrations/20231030115722_add_eg_schema/migration.sql create mode 100644 impl/c-qube/prisma/migrations/20231030115911_change_egschema_datatype/migration.sql diff --git a/impl/c-qube/prisma/migrations/20231030113226_add_metric_in_eg/migration.sql b/impl/c-qube/prisma/migrations/20231030113226_add_metric_in_eg/migration.sql new file mode 100644 index 00000000..22591ae7 --- /dev/null +++ b/impl/c-qube/prisma/migrations/20231030113226_add_metric_in_eg/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - Added the required column `metric` to the `EventGrammar` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "spec"."EventGrammar" ADD COLUMN "metric" TEXT NOT NULL; diff --git a/impl/c-qube/prisma/migrations/20231030115722_add_eg_schema/migration.sql b/impl/c-qube/prisma/migrations/20231030115722_add_eg_schema/migration.sql new file mode 100644 index 00000000..a70e7e32 --- /dev/null +++ b/impl/c-qube/prisma/migrations/20231030115722_add_eg_schema/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - Added the required column `egSchema` to the `EventGrammar` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "spec"."EventGrammar" ADD COLUMN "egSchema" JSONB NOT NULL; diff --git a/impl/c-qube/prisma/migrations/20231030115911_change_egschema_datatype/migration.sql b/impl/c-qube/prisma/migrations/20231030115911_change_egschema_datatype/migration.sql new file mode 100644 index 00000000..b0d49bd4 --- /dev/null +++ b/impl/c-qube/prisma/migrations/20231030115911_change_egschema_datatype/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "spec"."EventGrammar" ALTER COLUMN "egSchema" SET DATA TYPE TEXT; diff --git a/impl/c-qube/prisma/schema.prisma b/impl/c-qube/prisma/schema.prisma index d73dcd65..ed1af19c 100644 --- a/impl/c-qube/prisma/schema.prisma +++ b/impl/c-qube/prisma/schema.prisma @@ -72,6 +72,8 @@ model EventGrammar { file String? eventType EventType @default(INTERNAL) DatasetGrammar DatasetGrammar[] + metric String + egSchema String @@schema("spec") } diff --git a/impl/c-qube/src/services/csv-adapter/csv-adapter.service.ts b/impl/c-qube/src/services/csv-adapter/csv-adapter.service.ts index e206fcee..e1512022 100644 --- a/impl/c-qube/src/services/csv-adapter/csv-adapter.service.ts +++ b/impl/c-qube/src/services/csv-adapter/csv-adapter.service.ts @@ -27,6 +27,7 @@ import { } from './parser/dataset/dataset-grammar.helper'; import { createEventGrammarFromCSVDefinition, + getEGDefFromDB, getEGDefFromFile, } from './parser/event-grammar/event-grammar.service'; import { @@ -494,77 +495,77 @@ export class CsvAdapterService { for (let m = 0; m < compoundDatasetGrammars.length; m++) { promises.push( limit(() => - getEGDefFromFile(compoundDatasetGrammars[m].eventGrammarFile).then( - async (s) => { - const { - instrumentField, - }: { - eventGrammarDef: EventGrammarCSVFormat[]; - instrumentField: string; - } = s; - const compoundEventGrammar: EventGrammar = { - name: '', - description: '', - dimension: [], - instrument_field: instrumentField, - is_active: true, - schema: {}, - instrument: { - type: InstrumentType.COUNTER, - name: 'counter', - }, - }; - const events: Event[] = - await createCompoundDatasetDataToBeInserted( - compoundDatasetGrammars[m].eventGrammarFile.replace( - 'grammar', - 'data', - ), - compoundEventGrammar, - compoundDatasetGrammars[m], - ); - // Create Pipes - const pipe: Pipe = { - event: compoundEventGrammar, - transformer: defaultTransformers[0], - dataset: compoundDatasetGrammars[m], - }; - const transformContext: TransformerContext = { - dataset: compoundDatasetGrammars[m], - events: events, - isChainable: false, - pipeContext: {}, - }; - if (events && events.length > 0) { - const datasetUpdateRequest: DatasetUpdateRequest[] = - pipe.transformer.transformSync( - callback, - transformContext, - events, - ) as DatasetUpdateRequest[]; - - // console.log(datasetUpdateRequest.length, datasetUpdateRequest[0]); - - await this.datasetService - .processDatasetUpdateRequest(datasetUpdateRequest) - .then(() => { - this.logger.verbose( - `Ingested Compound Dataset without any error ${events.length} events for ${compoundDatasetGrammars[m].name}`, - ); - }) - .catch((e) => { - this.logger.verbose( - `Ingested Compound Dataset with error ${events.length} events for ${compoundDatasetGrammars[m].name}`, - ); - }); - } else { - console.error( - 'No relevant events for this dataset', - compoundDatasetGrammars[m].name, - ); - } - }, - ), + getEGDefFromDB( + compoundDatasetGrammars[m].eventGrammarFile, + this.prisma, + ).then(async (s) => { + const { + instrumentField, + }: { + eventGrammarDef: EventGrammarCSVFormat[]; + instrumentField: string; + } = s as any; + const compoundEventGrammar: EventGrammar = { + name: '', + description: '', + dimension: [], + instrument_field: instrumentField, + is_active: true, + schema: {}, + instrument: { + type: InstrumentType.COUNTER, + name: 'counter', + }, + }; + const events: Event[] = await createCompoundDatasetDataToBeInserted( + compoundDatasetGrammars[m].eventGrammarFile.replace( + 'grammar', + 'data', + ), + compoundEventGrammar, + compoundDatasetGrammars[m], + ); + // Create Pipes + const pipe: Pipe = { + event: compoundEventGrammar, + transformer: defaultTransformers[0], + dataset: compoundDatasetGrammars[m], + }; + const transformContext: TransformerContext = { + dataset: compoundDatasetGrammars[m], + events: events, + isChainable: false, + pipeContext: {}, + }; + if (events && events.length > 0) { + const datasetUpdateRequest: DatasetUpdateRequest[] = + pipe.transformer.transformSync( + callback, + transformContext, + events, + ) as DatasetUpdateRequest[]; + + // console.log(datasetUpdateRequest.length, datasetUpdateRequest[0]); + + await this.datasetService + .processDatasetUpdateRequest(datasetUpdateRequest) + .then(() => { + this.logger.verbose( + `Ingested Compound Dataset without any error ${events.length} events for ${compoundDatasetGrammars[m].name}`, + ); + }) + .catch((e) => { + this.logger.verbose( + `Ingested Compound Dataset with error ${events.length} events for ${compoundDatasetGrammars[m].name}`, + ); + }); + } else { + console.error( + 'No relevant events for this dataset', + compoundDatasetGrammars[m].name, + ); + } + }), ), ); } diff --git a/impl/c-qube/src/services/csv-adapter/parser/event-grammar/event-grammar.service.ts b/impl/c-qube/src/services/csv-adapter/parser/event-grammar/event-grammar.service.ts index e3c1493e..edf3eeae 100644 --- a/impl/c-qube/src/services/csv-adapter/parser/event-grammar/event-grammar.service.ts +++ b/impl/c-qube/src/services/csv-adapter/parser/event-grammar/event-grammar.service.ts @@ -15,6 +15,7 @@ import { FieldType, } from '../../types/parser'; import { readCSVFile } from '../utils/csvreader'; +import { PrismaService } from 'src/prisma.service'; export async function getEGDefFromFile(csvFilePath: string) { const [ @@ -39,6 +40,28 @@ export async function getEGDefFromFile(csvFilePath: string) { return { eventGrammarDef, instrumentField }; } +export async function getEGDefFromDB( + csvFilePath: string, + prisma: PrismaService, +) { + console.log('csvFilePath: ', csvFilePath); + const metrics = await prisma.eventGrammar.findMany({ + where: { + file: csvFilePath, + }, + select: { + egSchema: true, + metric: true, + }, + }); + console.log('metrics: ', metrics); + + return { + eventGrammarDef: metrics[0]?.egSchema, + instrumentField: metrics[0]?.metric, + }; +} + export const createEventGrammarFromCSVDefinition = async ( csvFilePath: string, dimensionFileBasePath: string, diff --git a/impl/c-qube/src/services/event/event.service.ts b/impl/c-qube/src/services/event/event.service.ts index 84c1adfe..f2064ce0 100644 --- a/impl/c-qube/src/services/event/event.service.ts +++ b/impl/c-qube/src/services/event/event.service.ts @@ -15,6 +15,10 @@ import { QueryBuilderService } from '../query-builder/query-builder.service'; import { DimensionService } from '../dimension/dimension.service'; import { DimensionMapping } from 'src/types/dataset'; import { DimensionGrammar } from 'src/types/dimension'; +import { + getEGDefFromDB, + getEGDefFromFile, +} from '../csv-adapter/parser/event-grammar/event-grammar.service'; @Injectable() export class EventService { @@ -65,6 +69,10 @@ export class EventService { await this.dimensionService.getDimensionGrammaModelByName( eventGrammar.dimension[0].dimension.name.name, ); + const { eventGrammarDef, instrumentField } = await getEGDefFromFile( + eventGrammar.file, + ); + return this.prisma.eventGrammar .create({ data: { @@ -82,6 +90,8 @@ export class EventService { }, }, ]), + metric: eventGrammar.instrument_field, + egSchema: JSON.stringify(eventGrammarDef), instrument: { connect: { name: 'COUNTER', //TODO: Change this to eventGrammar.instrument.name diff --git a/impl/c-qube/src/services/query-builder/query-builder.service.ts b/impl/c-qube/src/services/query-builder/query-builder.service.ts index e76e1ce8..06665f0b 100644 --- a/impl/c-qube/src/services/query-builder/query-builder.service.ts +++ b/impl/c-qube/src/services/query-builder/query-builder.service.ts @@ -110,7 +110,7 @@ export class QueryBuilderService { .slice(0, -2) .concat(uniqueStatements) .concat(');'); - console.log('sql:', createStatement); + // console.log('sql:', createStatement); // console.log('schema: ', schema); }