From 3211befcb294dceead4456af74a50c14423b2967 Mon Sep 17 00:00:00 2001 From: AleDore Date: Tue, 12 Mar 2024 15:30:24 +0100 Subject: [PATCH 1/3] [#IOPLT-368] Refactor DB Enrichment Datasource --- src/types/__tests__/enrichment.test.ts | 18 +++-- src/types/enrichment/enrichment.ts | 95 +++++++++++++++++++++++--- 2 files changed, 93 insertions(+), 20 deletions(-) diff --git a/src/types/__tests__/enrichment.test.ts b/src/types/__tests__/enrichment.test.ts index 4db4975..78bc623 100644 --- a/src/types/__tests__/enrichment.test.ts +++ b/src/types/__tests__/enrichment.test.ts @@ -1,6 +1,7 @@ import * as E from "fp-ts/Either"; import { pipe } from "fp-ts/lib/function"; -import { EnrichmentDataSource } from "../enrichment/enrichment"; +import { DBEnrichmentDataSource, DocumentDBParams, EnrichmentDataSource } from "../enrichment/enrichment"; +import { NonEmptyString } from "io-ts-types"; describe("DataEnrichment", () => { const aBlobStorageParams = { @@ -12,21 +13,18 @@ describe("DataEnrichment", () => { const aCosmosDBParams = { connectionString: "cosmos_db_connection_string", - query: "SELECT * FROM container", - partitionKey: "pk", - id: "id", + dbResourceKeyFieldName: "foo", + dbResourceName: "container", + queryType: "FIND_BY_KEY", + streamKeyFieldName: "foo" }; const aMongoDBParams = { - connectionString: "mongo_db_connection_string", - query: "db.collection.find({})", - id: "id", + ...aCosmosDBParams }; const aPostgresDBParams = { - connectionString: "postgres_db_connection_string", - query: "SELECT * FROM table", - id: "id", + ...aCosmosDBParams }; const anAPIParams = { diff --git a/src/types/enrichment/enrichment.ts b/src/types/enrichment/enrichment.ts index 05c221d..f18ce54 100644 --- a/src/types/enrichment/enrichment.ts +++ b/src/types/enrichment/enrichment.ts @@ -46,28 +46,103 @@ export const API = t.type({ const CommonDBParams = t.type({ connectionString: NonEmptyString, - id: NonEmptyString, - query: NonEmptyString, + dbResourceName: NonEmptyString, }); -const CosmosDBParams = t.intersection([ - CommonDBParams, +export const CommonFindDbParams = t.type({ + dbResourceKeyFieldName: NonEmptyString, + streamKeyFieldName: NonEmptyString, +}); + +export type CommonFindDbParams = t.TypeOf; + +export const RelationalModelFindByKeyParams = t.intersection([ + CommonFindDbParams, + t.type({ + queryType: t.literal("FIND_BY_KEY"), + }), +]); +export type RelationalModelFindByKeyParams = t.TypeOf< + typeof RelationalModelFindByKeyParams +>; + +export const DocumentModelQueryParams = t.intersection([ + CommonFindDbParams, + t.partial({ + dbResourcePkFieldName: NonEmptyString, + streamPkFieldName: NonEmptyString, + }), +]); + +export type DocumentModelQueryParams = t.TypeOf< + typeof DocumentModelQueryParams +>; + +export const DocumentModelFindByKeyQueryParams = t.intersection([ + DocumentModelQueryParams, + t.type({ + queryType: t.literal("FIND_BY_KEY"), + }), +]); + +export type DocumentModelFindByKeyQueryParams = t.TypeOf< + typeof DocumentModelFindByKeyQueryParams +>; + +export const DocumentModelVersionedQueryParams = t.intersection([ + DocumentModelQueryParams, + t.type({ + queryType: t.literal("FIND_LAST_VERSION"), + dbResourceVersionFieldName: NonEmptyString, + }), +]); + +export type DocumentModelVersionedQueryParams = t.TypeOf< + typeof DocumentModelVersionedQueryParams +>; + +export const RelationalModelVersionedQueryParams = t.intersection([ + CommonFindDbParams, t.type({ - partitionKey: NonEmptyString, + queryType: t.literal("FIND_LAST_VERSION"), + dbResourceVersionFieldName: NonEmptyString, }), ]); +export type RelationalModelVersionedQueryParams = t.TypeOf< + typeof RelationalModelVersionedQueryParams +>; + +export const DocumentDBParams = t.intersection([ + CommonDBParams, + t.union([ + DocumentModelFindByKeyQueryParams, + DocumentModelVersionedQueryParams, + ]), +]); + +export type DocumentDBParams = t.TypeOf; + +export const RelationalDBParams = t.intersection([ + CommonDBParams, + t.union([ + RelationalModelFindByKeyParams, + RelationalModelVersionedQueryParams, + ]), +]); +export type RelationalDBParams = t.TypeOf; -const DB = t.union([ - t.type({ type: t.literal("CosmosDB"), params: CosmosDBParams }), - t.type({ type: t.literal("MongoDB"), params: CommonDBParams }), - t.type({ type: t.literal("PosgresDB"), params: CommonDBParams }), +export const DBEnrichmentDataSource = t.union([ + t.type({ type: t.literal("CosmosDB"), params: DocumentDBParams }), + t.type({ type: t.literal("MongoDB"), params: DocumentDBParams }), + t.type({ type: t.literal("PosgresDB"), params: RelationalDBParams }), ]); +export type DBEnrichmentDataSource = t.TypeOf; export const EnrichmentDataSource = t.union([ BlobStorage, TableStorage, API, - DB, + DBEnrichmentDataSource, ]); export type EnrichmentDataSource = t.TypeOf; From 819d05a4d0001a3429aecd9b6be90f61aa7e24de Mon Sep 17 00:00:00 2001 From: AleDore Date: Tue, 12 Mar 2024 15:33:59 +0100 Subject: [PATCH 2/3] fix lint + add query selection --- src/types/__tests__/enrichment.test.ts | 3 +-- src/types/enrichment/enrichment.ts | 14 ++++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/types/__tests__/enrichment.test.ts b/src/types/__tests__/enrichment.test.ts index 78bc623..6401abd 100644 --- a/src/types/__tests__/enrichment.test.ts +++ b/src/types/__tests__/enrichment.test.ts @@ -1,7 +1,6 @@ import * as E from "fp-ts/Either"; import { pipe } from "fp-ts/lib/function"; -import { DBEnrichmentDataSource, DocumentDBParams, EnrichmentDataSource } from "../enrichment/enrichment"; -import { NonEmptyString } from "io-ts-types"; +import { EnrichmentDataSource } from "../enrichment/enrichment"; describe("DataEnrichment", () => { const aBlobStorageParams = { diff --git a/src/types/enrichment/enrichment.ts b/src/types/enrichment/enrichment.ts index f18ce54..e1202c2 100644 --- a/src/types/enrichment/enrichment.ts +++ b/src/types/enrichment/enrichment.ts @@ -1,6 +1,7 @@ /* eslint-disable sort-keys */ import { NonEmptyString } from "@pagopa/ts-commons/lib/strings"; import * as t from "io-ts"; +import { nonEmptyArray } from "io-ts-types"; const APIParams = t.type({ authentication: t.partial({ @@ -49,10 +50,15 @@ const CommonDBParams = t.type({ dbResourceName: NonEmptyString, }); -export const CommonFindDbParams = t.type({ - dbResourceKeyFieldName: NonEmptyString, - streamKeyFieldName: NonEmptyString, -}); +export const CommonFindDbParams = t.intersection([ + t.type({ + dbResourceKeyFieldName: NonEmptyString, + streamKeyFieldName: NonEmptyString, + }), + t.partial({ + selectFields: nonEmptyArray(NonEmptyString), + }), +]); export type CommonFindDbParams = t.TypeOf; From a43702874be981fd843fbb6fce0c370ceb8570c6 Mon Sep 17 00:00:00 2001 From: AleDore Date: Tue, 12 Mar 2024 15:42:38 +0100 Subject: [PATCH 3/3] remove nonEmptyArray --- src/types/enrichment/enrichment.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/types/enrichment/enrichment.ts b/src/types/enrichment/enrichment.ts index e1202c2..c0a6df0 100644 --- a/src/types/enrichment/enrichment.ts +++ b/src/types/enrichment/enrichment.ts @@ -1,7 +1,6 @@ /* eslint-disable sort-keys */ import { NonEmptyString } from "@pagopa/ts-commons/lib/strings"; import * as t from "io-ts"; -import { nonEmptyArray } from "io-ts-types"; const APIParams = t.type({ authentication: t.partial({ @@ -56,7 +55,7 @@ export const CommonFindDbParams = t.intersection([ streamKeyFieldName: NonEmptyString, }), t.partial({ - selectFields: nonEmptyArray(NonEmptyString), + selectFields: t.readonlyArray(NonEmptyString), }), ]);