diff --git a/api/package.json b/api/package.json index 2648963d..cf808119 100644 --- a/api/package.json +++ b/api/package.json @@ -79,7 +79,8 @@ "@trpc/server": "^10.18.0", "@types/pg": "^8.11.6", "jwt-decode": "^3.1.2", - "kysely": "^0.27.3", + "kysely": "^0.27.4", + "kysely-ctl": "^0.8.10", "pg": "^8.11.5", "semver": "^7.5.4", "tsafe": "^1.6.6", diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts index 79069476..e070d221 100644 --- a/api/scripts/load-git-repo-in-pg.ts +++ b/api/scripts/load-git-repo-in-pg.ts @@ -1,10 +1,11 @@ -import { Kysely } from "kysely"; +import { InsertObject, Kysely } from "kysely"; import { z } from "zod"; import { createGitDbApi, GitDbApiParams } from "../src/core/adapters/dbApi/createGitDbApi"; import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database"; import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect"; import { CompiledData } from "../src/core/ports/CompileData"; import { Db } from "../src/core/ports/DbApi"; +import { ExternalDataOrigin } from "../src/core/ports/GetSoftwareExternalData"; import SoftwareRow = Db.SoftwareRow; export type Params = { @@ -39,7 +40,7 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => { }); const compiledSoftwares = await gitDbApi.fetchCompiledData(); - await insertCompiledSoftwares(compiledSoftwares, pgDb); + await insertCompiledSoftwaresAndSoftwareExternalData(compiledSoftwares, pgDb); }; const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely) => { @@ -135,19 +136,15 @@ const insertInstances = async ({ instanceRows, db }: { instanceRows: Db.Instance console.info("Number of instances to insert : ", instanceRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("instances").execute(); + await trx .insertInto("instances") - .values( - instanceRows.map(row => ({ - ...row, - otherSoftwareWikidataIds: JSON.stringify(row.otherSoftwareWikidataIds) - })) - ) + .values(instanceRows.map(({ otherSoftwareWikidataIds, ...row }) => row)) .executeTakeFirst(); }); }; -const insertCompiledSoftwares = async ( +const insertCompiledSoftwaresAndSoftwareExternalData = async ( compiledSoftwares: CompiledData.Software<"private">[], pgDb: Kysely ) => { @@ -158,16 +155,51 @@ const insertCompiledSoftwares = async ( await trx .insertInto("compiled_softwares") .values( - compiledSoftwares.map(software => ({ - softwareId: software.id, - serviceProviders: JSON.stringify(software.serviceProviders), - softwareExternalData: JSON.stringify(software.softwareExternalData), - similarExternalSoftwares: JSON.stringify(software.similarExternalSoftwares), - parentWikidataSoftware: JSON.stringify(software.parentWikidataSoftware), - comptoirDuLibreSoftware: JSON.stringify(software.comptoirDuLibreSoftware), - annuaireCnllServiceProviders: JSON.stringify(software.annuaireCnllServiceProviders), - latestVersion: JSON.stringify(software.latestVersion) - })) + compiledSoftwares.map( + (software): InsertObject => ({ + softwareId: software.id, + serviceProviders: JSON.stringify(software.serviceProviders), + comptoirDuLibreSoftware: JSON.stringify(software.comptoirDuLibreSoftware), + annuaireCnllServiceProviders: JSON.stringify(software.annuaireCnllServiceProviders), + latestVersion: JSON.stringify(software.latestVersion) + }) + ) + ) + .executeTakeFirst(); + + await trx.deleteFrom("software_external_datas").execute(); + await trx + .insertInto("software_external_datas") + .values( + compiledSoftwares + .filter( + ( + software + ): software is CompiledData.Software.Private & { + softwareExternalData: { + externalId: string; + externalDataOrigin: ExternalDataOrigin; + }; + } => + software.softwareExternalData?.externalId !== undefined && + software.softwareExternalData?.externalDataOrigin !== undefined + ) + .map( + (software): InsertObject => ({ + externalId: software.softwareExternalData.externalId, + externalDataOrigin: software.softwareExternalData.externalDataOrigin, + developers: JSON.stringify(software.softwareExternalData?.developers ?? []), + label: JSON.stringify(software.softwareExternalData?.label ?? {}), + description: JSON.stringify(software.softwareExternalData?.description ?? {}), + isLibreSoftware: software.softwareExternalData?.isLibreSoftware ?? false, + logoUrl: software.softwareExternalData?.logoUrl ?? null, + framaLibreId: software.softwareExternalData?.framaLibreId ?? null, + websiteUrl: software.softwareExternalData?.websiteUrl ?? null, + sourceUrl: software.softwareExternalData?.sourceUrl ?? null, + documentationUrl: software.softwareExternalData?.documentationUrl ?? null, + license: software.softwareExternalData?.license ?? null + }) + ) ) .executeTakeFirst(); }); diff --git a/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts b/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts index 23682175..32b592fa 100644 --- a/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts +++ b/api/src/core/adapters/dbApi/kysely/createPgDbApi.ts @@ -1,30 +1,42 @@ -import { Kysely } from "kysely"; +import { Kysely, sql } from "kysely"; import type { Equals } from "tsafe"; import { assert } from "tsafe/assert"; import { CompiledData } from "../../../ports/CompileData"; import { Db } from "../../../ports/DbApi"; -import { Software, SoftwareFormData } from "../../../usecases/readWriteSillData"; +import { + ExternalDataOrigin, + ParentSoftwareExternalData, + SoftwareExternalData +} from "../../../ports/GetSoftwareExternalData"; +import { + DeclarationFormData, + Instance, + InstanceFormData, + Software, + SoftwareFormData +} from "../../../usecases/readWriteSillData"; import { Database } from "./kysely.database"; -import { castSql, jsonBuildObject } from "./kysely.utils"; -import SimilarSoftware = Software.SimilarSoftware; +import { jsonBuildObject } from "./kysely.utils"; type Agent = { id: number; email: string; organization: string }; type WithAgent = { agent: Agent }; type NewDbApi = { software: { - create: (params: { formData: SoftwareFormData } & WithAgent) => Promise; + create: ( + params: { formData: SoftwareFormData; externalDataOrigin: ExternalDataOrigin } & WithAgent + ) => Promise; update: (params: { softwareSillId: number; formData: SoftwareFormData } & WithAgent) => Promise; getAll: () => Promise; unreference: () => {}; }; instance: { - create: () => {}; - update: () => {}; - getAll: () => {}; + create: (params: { fromData: InstanceFormData } & WithAgent) => Promise; + update: (params: { fromData: InstanceFormData; instanceId: number } & WithAgent) => Promise; + getAll: () => Promise; }; agent: { - createUserOrReferent: () => {}; + createUserOrReferent: (params: { fromData: DeclarationFormData; softwareName: string }) => Promise; removeUserOrReferent: () => {}; updateIsProfilePublic: () => {}; updateAbout: () => {}; @@ -42,7 +54,7 @@ export type PgDbApi = ReturnType; export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { return { software: { - create: async ({ formData, agent }) => { + create: async ({ formData, externalDataOrigin, agent }) => { const { softwareName, softwareDescription, @@ -81,6 +93,7 @@ export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { isPresentInSupportContract: isPresentInSupportContract, similarSoftwareExternalDataIds: JSON.stringify(similarSoftwareExternalDataIds), externalId: externalId, + externalDataOrigin: externalDataOrigin, comptoirDuLibreId: comptoirDuLibreId, softwareType: JSON.stringify(softwareType), catalogNumeriqueGouvFrId: undefined, @@ -146,10 +159,36 @@ export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { getAll: (): Promise => db .selectFrom("softwares as s") + .leftJoin("software_external_datas as ext", "ext.externalId", "s.externalId") .leftJoin("compiled_softwares as cs", "cs.softwareId", "s.id") + .leftJoin( + "software_external_datas as parentExt", + "s.parentSoftwareWikidataId", + "parentExt.externalId" + ) + .leftJoin( + "softwares__similar_software_external_datas", + "softwares__similar_software_external_datas.softwareId", + "s.id" + ) + .leftJoin( + "software_external_datas as similarExt", + "softwares__similar_software_external_datas.similarExternalId", + "similarExt.externalId" + ) + .groupBy([ + "s.id", + "cs.softwareId", + "cs.annuaireCnllServiceProviders", + "cs.comptoirDuLibreSoftware", + "cs.latestVersion", + "cs.serviceProviders", + "ext.externalId", + "parentExt.externalId" + ]) .select([ - "s.logoUrl", "s.id as softwareId", + "s.logoUrl", "s.name as softwareName", "s.description as softwareDescription", "cs.serviceProviders", @@ -173,10 +212,56 @@ export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { "s.externalId", "s.externalDataOrigin", "s.softwareType", - "cs.parentWikidataSoftware", - "cs.similarExternalSoftwares as similarSoftwares", + ({ ref, ...qb }) => + qb + .case() + .when("parentExt.externalId", "is not", null) + .then( + jsonBuildObject({ + externalId: ref("ext.externalId"), + label: ref("ext.label"), + description: ref("ext.description") + }).$castTo() + ) + .else(null) + .end() + .as("parentExternalData"), + "s.similarSoftwareExternalDataIds", "s.keywords", - "softwareExternalData" + + ({ ref }) => + jsonBuildObject({ + externalId: ref("ext.externalId"), + externalDataOrigin: ref("ext.externalDataOrigin"), + developers: ref("ext.developers"), + label: ref("ext.label"), + description: ref("ext.description"), + isLibreSoftware: ref("ext.isLibreSoftware"), + logoUrl: ref("ext.logoUrl"), + framaLibreId: ref("ext.framaLibreId"), + websiteUrl: ref("ext.websiteUrl"), + sourceUrl: ref("ext.sourceUrl"), + documentationUrl: ref("ext.documentationUrl") + }).as("softwareExternalData"), + // ({ fn }) => fn.jsonAgg("similarExt").distinct().as("similarExternalSoftwares") + ({ ref, fn }) => + fn + .coalesce( + fn + .jsonAgg( + jsonBuildObject({ + isInSill: sql`false`, + externalId: ref("similarExt.externalId"), + label: ref("similarExt.label"), + description: ref("similarExt.description"), + isLibreSoftware: ref("similarExt.isLibreSoftware"), + externalDataOrigin: ref("similarExt.externalDataOrigin") + }).$castTo() + ) + .filterWhere("similarExt.externalId", "is not", null), + sql<[]>`'[]'` + ) + .as("similarExternalSoftwares") ]) .execute() .then(softwares => @@ -184,52 +269,142 @@ export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { ({ testUrls, serviceProviders, - similarSoftwares, - softwareExternalData, + similarSoftwareExternalDataIds, + parentExternalData, updateTime, addedTime, + softwareExternalData, + similarExternalSoftwares, ...software - }): Software => ({ - ...convertNullValuesToUndefined(software), - updateTime: new Date(+updateTime).getTime(), - addedTime: new Date(+addedTime).getTime(), - serviceProviders: serviceProviders ?? [], - similarSoftwares: - (similarSoftwares ?? []).map( - (s): SimilarSoftware => ({ - softwareName: - typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - softwareDescription: - typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, - isInSill: true // TODO: check if this is true - }) - ) ?? [], - userAndReferentCountByOrganization: {}, - authors: (softwareExternalData?.developers ?? []).map(dev => ({ - authorName: dev.name, - authorUrl: `https://www.wikidata.org/wiki/${dev.id}` - })), - officialWebsiteUrl: - softwareExternalData?.websiteUrl ?? - software.comptoirDuLibreSoftware?.external_resources.website ?? - undefined, - codeRepositoryUrl: - softwareExternalData?.sourceUrl ?? - software.comptoirDuLibreSoftware?.external_resources.repository ?? - undefined, - documentationUrl: softwareExternalData?.documentationUrl, - comptoirDuLibreServiceProviderCount: - software.comptoirDuLibreSoftware?.providers.length ?? 0, - testUrl: testUrls[0]?.url - }) + }): Software => { + return { + ...convertNullValuesToUndefined(software), + updateTime: new Date(+updateTime).getTime(), + addedTime: new Date(+addedTime).getTime(), + serviceProviders: serviceProviders ?? [], + similarSoftwares: similarExternalSoftwares, + // (similarSoftwares ?? []).map( + // (s): SimilarSoftware => ({ + // softwareName: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // softwareDescription: + // typeof s.label === "string" ? s.label : Object.values(s.label)[0]!, + // isInSill: true // TODO: check if this is true + // }) + // ) ?? [], + userAndReferentCountByOrganization: {}, + authors: (softwareExternalData?.developers ?? []).map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), + officialWebsiteUrl: + softwareExternalData?.websiteUrl ?? + software.comptoirDuLibreSoftware?.external_resources.website ?? + undefined, + codeRepositoryUrl: + softwareExternalData?.sourceUrl ?? + software.comptoirDuLibreSoftware?.external_resources.repository ?? + undefined, + documentationUrl: softwareExternalData?.documentationUrl ?? undefined, + comptoirDuLibreServiceProviderCount: + software.comptoirDuLibreSoftware?.providers.length ?? 0, + testUrl: testUrls[0]?.url, + parentWikidataSoftware: parentExternalData ?? undefined + }; + } ) ), unreference: async () => {} }, instance: { - create: async () => {}, - update: async () => {}, - getAll: async () => {} + create: async ({ fromData, agent }) => { + const { + mainSoftwareSillId, + organization, + targetAudience, + publicUrl, + otherSoftwareWikidataIds, + ...rest + } = fromData; + assert>(); + + const now = Date.now(); + await db + .insertInto("instances") + .values({ + addedByAgentEmail: agent.email, + updateTime: now, + referencedSinceTime: now, + mainSoftwareSillId, + organization, + targetAudience, + publicUrl + }) + .execute(); + }, + update: async ({ fromData, instanceId }) => { + const { + mainSoftwareSillId, + organization, + targetAudience, + publicUrl, + otherSoftwareWikidataIds, + ...rest + } = fromData; + assert>(); + + const now = Date.now(); + await db + .updateTable("instances") + .set({ + updateTime: now, + referencedSinceTime: now, + mainSoftwareSillId, + organization, + targetAudience, + publicUrl + }) + .where("id", "=", instanceId) + .execute(); + }, + getAll: async () => + db + .selectFrom("instances as i") + .leftJoin("instances__other_external_softwares as ioes", "ioes.instanceId", "i.id") + .leftJoin("software_external_datas as ext", "ext.externalId", "ioes.externalId") + .select([ + "i.id", + "i.mainSoftwareSillId", + "i.organization", + "i.targetAudience", + "i.publicUrl", + ({ fn }) => + fn + .jsonAgg("ext") + .distinct() + .$castTo() + .as("otherWikidataSoftwares") + // ({ ref, fn }) => + // fn + // .jsonAgg( + // jsonBuildObject({ + // externalId: ref("ext.externalId"), + // label: ref("ext.label"), + // description: ref("ext.description") + // }).$castTo() + // ) + // .as("otherWikidataSoftwares") + ]) + .execute() + .then(instances => + instances.map( + (instance): Instance => ({ + ...instance, + publicUrl: instance.publicUrl ?? undefined, + otherWikidataSoftwares: instance.otherWikidataSoftwares + }) + ) + ) }, agent: { createUserOrReferent: async () => {}, @@ -244,9 +419,6 @@ export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { getTotalReferentCount: async () => {} }, getCompiledDataPrivate: async (): Promise> => { - const builder = getQueryBuilder(db); - console.log(builder.compile().sql); - console.time("agentById query"); const agentById: Record = await db .selectFrom("agents") @@ -262,16 +434,28 @@ export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { .leftJoin("software_referents as referents", "s.id", "referents.softwareId") .leftJoin("software_users as users", "s.id", "users.softwareId") .leftJoin("instances", "s.id", "instances.mainSoftwareSillId") + .leftJoin("software_external_datas as ext", "ext.externalId", "s.externalId") + .leftJoin("software_external_datas as parentExt", "parentExt.externalId", "s.parentSoftwareWikidataId") + .leftJoin( + "softwares__similar_software_external_datas", + "softwares__similar_software_external_datas.softwareId", + "s.id" + ) + .leftJoin( + "software_external_datas as similarExt", + "softwares__similar_software_external_datas.similarExternalId", + "similarExt.externalId" + ) .groupBy([ "s.id", "csft.softwareId", "csft.annuaireCnllServiceProviders", "csft.comptoirDuLibreSoftware", "csft.latestVersion", - "csft.parentWikidataSoftware", "csft.serviceProviders", - "csft.similarExternalSoftwares", - "csft.softwareExternalData" + "parentExt.externalId" + // "csft.similarExternalSoftwares", + // "csft.softwareExternalData" ]) .select([ "s.id", @@ -301,10 +485,45 @@ export const createKyselyPgDbApi = (db: Kysely): NewDbApi => { "csft.annuaireCnllServiceProviders", "csft.comptoirDuLibreSoftware", "csft.latestVersion", - "csft.parentWikidataSoftware", "csft.serviceProviders", - "csft.similarExternalSoftwares", - "csft.softwareExternalData", + // "csft.parentWikidataSoftware", + // "csft.similarExternalSoftwares", + ({ ref }) => + jsonBuildObject({ + externalId: ref("parentExt.externalId"), + label: ref("parentExt.label"), + description: ref("parentExt.description") + }) + .$castTo() + .as("parentWikidataSoftware"), + ({ ref }) => + jsonBuildObject({ + externalId: ref("ext.externalId"), + externalDataOrigin: ref("ext.externalDataOrigin"), + developers: ref("ext.developers"), + label: ref("ext.label"), + description: ref("ext.description"), + isLibreSoftware: ref("ext.isLibreSoftware"), + logoUrl: ref("ext.logoUrl"), + framaLibreId: ref("ext.framaLibreId"), + websiteUrl: ref("ext.websiteUrl"), + sourceUrl: ref("ext.sourceUrl"), + documentationUrl: ref("ext.documentationUrl") + }) + .$castTo() + .as("softwareExternalData"), + ({ ref, fn }) => + fn + .jsonAgg( + jsonBuildObject({ + externalId: ref("similarExt.externalId"), + label: ref("similarExt.label"), + description: ref("similarExt.description"), + isLibreSoftware: ref("similarExt.isLibreSoftware"), + externalDataOrigin: ref("similarExt.externalDataOrigin") + }).$castTo() + ) + .as("similarExternalSoftwares"), ({ fn }) => fn.jsonAgg("users").distinct().as("users"), ({ fn }) => fn.jsonAgg("referents").distinct().as("referents"), ({ fn }) => fn.jsonAgg("instances").distinct().as("instances") @@ -374,100 +593,3 @@ const convertNullValuesToUndefined = >( obj: T ): { [K in keyof T]: null extends T[K] ? Exclude | undefined : T[K] } => Object.fromEntries(Object.entries(obj).map(([key, value]) => [key, value === null ? undefined : value])) as any; - -// ----------- common ----------- -// annuaireCnllServiceProviders -// catalogNumeriqueGouvFrId -// categories -// comptoirDuLibreSoftware -// dereferencing -// description -// doRespectRgaa -// externalDataOrigin -// externalId -// generalInfoMd -// id -// isFromFrenchPublicService -// isPresentInSupportContract -// isStillInObservation -// keywords -// latestVersion -// license -// logoUrl -// name -// parentWikidataSoftware -// referencedSinceTime -// serviceProviders -// similarExternalSoftwares -// softwareExternalData -// softwareType -// testUrls -// updateTime -// versionMin -// workshopUrls -// -// ----------- private ----------- -// addedByAgentEmail -// users -// referents -// instances -// -// ----------- public ----------- -// userAndReferentCountByOrganization -// hasExpertReferent -// instances - -const getQueryBuilder = (db: Kysely) => - db - .selectFrom("softwares as s") - .leftJoin("compiled_softwares as csft", "csft.softwareId", "s.id") - .leftJoin("software_referents as referents", "s.id", "referents.softwareId") - .leftJoin("software_users as users", "s.id", "users.softwareId") - .leftJoin("instances", "s.id", "instances.mainSoftwareSillId") - .groupBy([ - "s.id", - "csft.softwareId", - "csft.annuaireCnllServiceProviders", - "csft.comptoirDuLibreSoftware", - "csft.latestVersion", - "csft.parentWikidataSoftware", - "csft.serviceProviders", - "csft.similarExternalSoftwares", - "csft.softwareExternalData" - ]) - .select([ - "s.id", - "s.addedByAgentEmail", - "s.catalogNumeriqueGouvFrId", - "s.categories", - "s.dereferencing", - "s.description", - "s.doRespectRgaa", - "s.externalDataOrigin", - "s.externalId", - "s.generalInfoMd", - "s.isFromFrenchPublicService", - "s.isPresentInSupportContract", - "s.isStillInObservation", - "s.keywords", - "s.license", - "s.logoUrl", - "s.name", - "s.referencedSinceTime", - "s.softwareType", - "s.testUrls", - "s.updateTime", - "s.versionMin", - "s.workshopUrls", - "csft.softwareId as externalDataSoftwareId", - "csft.annuaireCnllServiceProviders", - "csft.comptoirDuLibreSoftware", - "csft.latestVersion", - "csft.parentWikidataSoftware", - "csft.serviceProviders", - "csft.similarExternalSoftwares", - "csft.softwareExternalData", - ({ fn }) => fn.jsonAgg("users").distinct().as("users"), - ({ fn }) => fn.jsonAgg("referents").distinct().as("referents"), - ({ fn }) => fn.jsonAgg("instances").distinct().as("instances") - ]); diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index 14bbdcaa..b6329c18 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -1,12 +1,14 @@ import { Generated, JSONColumnType } from "kysely"; -import type { PartialNoOptional } from "../../../../tools/PartialNoOptional"; export type Database = { agents: AgentsTable; software_referents: SoftwareReferentsTable; software_users: SoftwareUsersTable; instances: InstancesTable; + instances__other_external_softwares: InstancesOtherExternalSoftwaresTable; softwares: SoftwaresTable; + software_external_datas: SoftwareExternalDatasTable; + softwares__similar_software_external_datas: SimilarExternalSoftwareExternalDataTable; compiled_softwares: CompiledSoftwaresTable; }; @@ -38,17 +40,50 @@ type SoftwareUsersTable = { }; type InstancesTable = { - id: number; + id: Generated; mainSoftwareSillId: number; organization: string; targetAudience: string; publicUrl: string | null; - otherSoftwareWikidataIds: JSONColumnType; addedByAgentEmail: string; referencedSinceTime: number; updateTime: number; }; +type InstancesOtherExternalSoftwaresTable = { + instanceId: number; + externalId: ExternalId; +}; + +type ExternalId = string; +type ExternalDataOrigin = "wikidata" | "HAL"; +type LocalizedString = Partial>; + +type SimilarExternalSoftwareExternalDataTable = { + softwareId: number; + similarExternalId: ExternalId; +}; + +type SoftwareExternalDatasTable = { + externalId: ExternalId; + externalDataOrigin: ExternalDataOrigin; + developers: JSONColumnType< + { + name: string; + id: string; + }[] + >; + label: string | JSONColumnType; + description: string | JSONColumnType; + isLibreSoftware: boolean; + logoUrl: string | null; + framaLibreId: string | null; + websiteUrl: string | null; + sourceUrl: string | null; + documentationUrl: string | null; + license: string | null; +}; + type SoftwareType = | { type: "cloud" } | { type: "stack" } @@ -69,11 +104,11 @@ type SoftwaresTable = { lastRecommendedVersion?: string; }> | null; isStillInObservation: boolean; - parentSoftwareWikidataId: string | null; doRespectRgaa: boolean | null; isFromFrenchPublicService: boolean; isPresentInSupportContract: boolean; similarSoftwareExternalDataIds: JSONColumnType; + parentSoftwareWikidataId: string | null; externalId: string | null; externalDataOrigin: "wikidata" | "HAL" | null; comptoirDuLibreId: number | null; @@ -144,37 +179,9 @@ type ServiceProvider = { siren?: string; }; -type ExternalDataDeveloper = { - name: string; - id: string; -}; - -type LocalizedString = string | Partial>; - -type SoftwareExternalData = { - externalId: string; - externalDataOrigin: "wikidata" | "HAL"; - developers: ExternalDataDeveloper[]; - label: LocalizedString; - description: LocalizedString; - isLibreSoftware: boolean; -} & PartialNoOptional<{ - logoUrl: string; - framaLibreId: string; - websiteUrl: string; - sourceUrl: string; - documentationUrl: string; - license: string; -}>; - type CompiledSoftwaresTable = { softwareId: number; serviceProviders: JSONColumnType; - softwareExternalData: JSONColumnType | null; - similarExternalSoftwares: JSONColumnType< - Pick[] - >; - parentWikidataSoftware: JSONColumnType> | null; comptoirDuLibreSoftware: JSONColumnType | null; annuaireCnllServiceProviders: JSONColumnType< { diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts index 9bdb377f..2729879f 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1717162141365_create-initial-tables.ts @@ -1,6 +1,7 @@ -import { Kysely } from "kysely"; +import { Kysely, sql } from "kysely"; export async function up(db: Kysely): Promise { + await db.schema.createType("external_data_origin_type").asEnum(["wikidata", "HAL"]).execute(); await db.schema .createTable("agents") .addColumn("id", "serial", col => col.primaryKey()) @@ -12,32 +13,57 @@ export async function up(db: Kysely): Promise { await db.schema .createTable("softwares") + // from form .addColumn("id", "serial", col => col.primaryKey()) + .addColumn("softwareType", "jsonb", col => col.notNull()) + .addColumn("externalId", "text") + .addColumn("externalDataOrigin", sql`external_data_origin_type`) + .addColumn("comptoirDuLibreId", "integer") .addColumn("name", "text", col => col.notNull()) .addColumn("description", "text", col => col.notNull()) - .addColumn("referencedSinceTime", "bigint", col => col.notNull()) - .addColumn("updateTime", "bigint", col => col.notNull()) - .addColumn("dereferencing", "jsonb") - .addColumn("isStillInObservation", "boolean", col => col.notNull()) - .addColumn("parentSoftwareWikidataId", "text") - .addColumn("doRespectRgaa", "boolean") - .addColumn("isFromFrenchPublicService", "boolean", col => col.notNull()) + .addColumn("license", "text", col => col.notNull()) + .addColumn("versionMin", "text", col => col.notNull()) .addColumn("isPresentInSupportContract", "boolean", col => col.notNull()) + .addColumn("isFromFrenchPublicService", "boolean", col => col.notNull()) + .addColumn("logoUrl", "text") + .addColumn("keywords", "jsonb", col => col.notNull()) + .addColumn("doRespectRgaa", "boolean") + // from ??? + .addColumn("isStillInObservation", "boolean", col => col.notNull()) .addColumn("similarSoftwareExternalDataIds", "jsonb") - .addColumn("externalId", "text") - .addColumn("externalDataOrigin", "text") - .addColumn("comptoirDuLibreId", "integer") - .addColumn("license", "text", col => col.notNull()) - .addColumn("softwareType", "jsonb", col => col.notNull()) + .addColumn("parentSoftwareWikidataId", "text") .addColumn("catalogNumeriqueGouvFrId", "text") - .addColumn("versionMin", "text", col => col.notNull()) .addColumn("workshopUrls", "jsonb", col => col.notNull()) .addColumn("testUrls", "jsonb", col => col.notNull()) .addColumn("categories", "jsonb", col => col.notNull()) .addColumn("generalInfoMd", "text") .addColumn("addedByAgentEmail", "text", col => col.notNull()) + .addColumn("dereferencing", "jsonb") + .addColumn("referencedSinceTime", "bigint", col => col.notNull()) + .addColumn("updateTime", "bigint", col => col.notNull()) + .execute(); + + await db.schema + .createTable("software_external_datas") + .addColumn("externalId", "text", col => col.primaryKey()) + .addColumn("externalDataOrigin", sql`external_data_origin_type`, col => col.notNull()) + .addColumn("developers", "jsonb", col => col.notNull()) + .addColumn("label", "jsonb", col => col.notNull()) + .addColumn("description", "jsonb", col => col.notNull()) + .addColumn("isLibreSoftware", "boolean", col => col.notNull()) .addColumn("logoUrl", "text") - .addColumn("keywords", "jsonb", col => col.notNull()) + .addColumn("framaLibreId", "text") + .addColumn("websiteUrl", "text") + .addColumn("sourceUrl", "text") + .addColumn("documentationUrl", "text") + .addColumn("license", "text") + .execute(); + + await db.schema + .createTable("softwares__similar_software_external_datas") + .addColumn("softwareId", "integer", col => col.notNull().references("softwares.id").onDelete("cascade")) + .addColumn("similarExternalId", "text", col => col.notNull()) + .addUniqueConstraint("uniq_software_id__similar_software_external_id", ["softwareId", "similarExternalId"]) .execute(); await db.schema @@ -61,22 +87,24 @@ export async function up(db: Kysely): Promise { await db.schema .createTable("instances") - .addColumn("id", "integer", col => col.notNull()) + .addColumn("id", "serial", col => col.primaryKey()) .addColumn("mainSoftwareSillId", "integer", col => col.notNull().references("softwares.id").onDelete("cascade")) .addColumn("addedByAgentEmail", "text", col => col.notNull()) .addColumn("organization", "text", col => col.notNull()) .addColumn("targetAudience", "text", col => col.notNull()) .addColumn("publicUrl", "text") - .addColumn("otherSoftwareWikidataIds", "jsonb") .addColumn("referencedSinceTime", "bigint", col => col.notNull()) .addColumn("updateTime", "bigint", col => col.notNull()) .execute(); } export async function down(db: Kysely): Promise { + await db.schema.dropTable("softwares__similar_software_external_datas").execute(); + await db.schema.dropTable("software_external_datas").execute(); await db.schema.dropTable("instances").execute(); await db.schema.dropTable("software_referents").execute(); await db.schema.dropTable("software_users").execute(); await db.schema.dropTable("softwares").execute(); await db.schema.dropTable("agents").execute(); + await db.schema.dropType("external_data_origin_type").execute(); } diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts index 7ad163a7..14eb2765 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts @@ -5,9 +5,6 @@ export async function up(db: Kysely): Promise { .createTable("compiled_softwares") .addColumn("softwareId", "integer", col => col.notNull().references("softwares.id").onDelete("cascade")) .addColumn("serviceProviders", "jsonb", col => col.notNull()) - .addColumn("softwareExternalData", "jsonb") - .addColumn("similarExternalSoftwares", "jsonb", col => col.notNull()) - .addColumn("parentWikidataSoftware", "jsonb") .addColumn("comptoirDuLibreSoftware", "jsonb") .addColumn("annuaireCnllServiceProviders", "jsonb") .addColumn("latestVersion", "jsonb") diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts b/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts index 40c41f65..a5a48ea8 100644 --- a/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts +++ b/api/src/core/adapters/dbApi/kysely/migrations/1720187269028_add-indexes.ts @@ -1,5 +1,6 @@ import type { Kysely } from "kysely"; +const softwares_externalIdIdx = "softwares__externalId_idx"; const compiledSoftwares_SoftwareIdIdx = "compiled_softwares__softwareId_idx"; const compiledSoftwares_GroupByIdx = "compiled_softwares_group_by_idx"; const softwareReferents_softwareIdIdx = "softwareReferents_software_idx"; @@ -7,6 +8,7 @@ const softwareUsers_softwareIdIdx = "softwareUsers_software_idx"; const instances_mainSoftwareSillIdIdx = "instances_mainSoftwareSillId_idx"; export async function up(db: Kysely): Promise { + await db.schema.createIndex(softwares_externalIdIdx).on("softwares").column("externalId").execute(); await db.schema .createIndex(compiledSoftwares_SoftwareIdIdx) .on("compiled_softwares") @@ -30,14 +32,12 @@ export async function up(db: Kysely): Promise { .column("annuaireCnllServiceProviders") .column("comptoirDuLibreSoftware") .column("latestVersion") - .column("parentWikidataSoftware") .column("serviceProviders") - .column("similarExternalSoftwares") - .column("softwareExternalData") .execute(); } export async function down(db: Kysely): Promise { + await db.schema.dropIndex(softwares_externalIdIdx).execute(); await db.schema.dropIndex(compiledSoftwares_SoftwareIdIdx).execute(); await db.schema.dropIndex(softwareReferents_softwareIdIdx).execute(); await db.schema.dropIndex(softwareUsers_softwareIdIdx).execute(); diff --git a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts index fdf6c8be..7b48135d 100644 --- a/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts +++ b/api/src/core/adapters/dbApi/kysely/pgDbApi.integration.test.ts @@ -1,19 +1,20 @@ -import { Kysely, sql } from "kysely"; -import { beforeEach, describe, it, expect } from "vitest"; +import { Kysely } from "kysely"; +import { beforeEach, describe, expect, it } from "vitest"; import { expectToEqual } from "../../../../tools/test.helpers"; -import { CompiledData } from "../../../ports/CompileData"; +import { SoftwareExternalData } from "../../../ports/GetSoftwareExternalData"; import { SoftwareFormData } from "../../../usecases/readWriteSillData"; import { createKyselyPgDbApi, PgDbApi } from "./createPgDbApi"; import { Database } from "./kysely.database"; import { createPgDialect } from "./kysely.dialect"; +const externalId = "external-id-111"; const softwareFormData: SoftwareFormData = { comptoirDuLibreId: 50, doRespectRgaa: true, externalId: "external-id-111", isFromFrenchPublicService: false, isPresentInSupportContract: true, - similarSoftwareExternalDataIds: ["external-id-222"], + similarSoftwareExternalDataIds: [externalId], softwareDescription: "Super software", softwareKeywords: ["bob", "l'éponge"], softwareLicense: "MIT", @@ -40,30 +41,63 @@ describe("pgDbApi", () => { beforeEach(async () => { dbApi = createKyselyPgDbApi(db); await db.deleteFrom("softwares").execute(); + await db.deleteFrom("software_external_datas").execute(); }); describe("getCompiledDataPrivate", () => { it("gets private compiled data", async () => { const compiledDataPrivate = await dbApi.getCompiledDataPrivate(); const { users, referents, instances, ...firstSoftware } = compiledDataPrivate[0]; - // console.log(firstSoftware); + console.log(firstSoftware); // - // console.log(`Users n = ${users?.length} : `, users); - // console.log(`Referents n = ${referents?.length} : `, referents); - // console.log(`Instances n = ${instances?.length} : `, instances); + console.log(`Users n = ${users?.length} : `, users); + console.log(`Referents n = ${referents?.length} : `, referents); + console.log(`Instances n = ${instances?.length} : `, instances); expect(compiledDataPrivate).toHaveLength(100); }); }); describe("software", () => { it("creates a software, than gets it with getAll", async () => { + const softwareExternalData: SoftwareExternalData = { + externalId, + externalDataOrigin: "wikidata", + developers: [{ name: "Bob", id: "bob" }], + label: { en: "Some software" }, + description: { en: "Some software description" }, + isLibreSoftware: true, + logoUrl: "https://example.com/logo.png", + framaLibreId: "", + websiteUrl: "https://example.com", + sourceUrl: "https://example.com/source", + documentationUrl: "https://example.com/documentation", + license: "MIT" + }; + + await db + .insertInto("software_external_datas") + .values({ + ...softwareExternalData, + developers: JSON.stringify(softwareExternalData.developers), + label: JSON.stringify(softwareExternalData.label), + description: JSON.stringify(softwareExternalData.description), + isLibreSoftware: softwareExternalData.isLibreSoftware, + framaLibreId: softwareExternalData.framaLibreId, + websiteUrl: softwareExternalData.websiteUrl, + sourceUrl: softwareExternalData.sourceUrl, + documentationUrl: softwareExternalData.documentationUrl, + license: softwareExternalData.license + }) + .execute(); + await dbApi.software.create({ formData: softwareFormData, agent: { id: 1, email: "test@test.com", organization: "test-orga" - } + }, + externalDataOrigin: "wikidata" }); const softwares = await dbApi.software.getAll(); @@ -72,20 +106,23 @@ describe("pgDbApi", () => { addedTime: expect.any(Number), updateTime: expect.any(Number), annuaireCnllServiceProviders: undefined, - authors: [], + authors: softwareExternalData.developers.map(dev => ({ + authorName: dev.name, + authorUrl: `https://www.wikidata.org/wiki/${dev.id}` + })), categories: [], - codeRepositoryUrl: undefined, + codeRepositoryUrl: softwareExternalData.sourceUrl, comptoirDuLibreId: 50, comptoirDuLibreServiceProviderCount: 0, dereferencing: undefined, - documentationUrl: undefined, - externalDataOrigin: undefined, - externalId: "external-id-111", + documentationUrl: softwareExternalData.documentationUrl, + externalDataOrigin: "wikidata", + externalId, keywords: ["bob", "l'éponge"], latestVersion: undefined, license: "MIT", logoUrl: "https://example.com/logo.png", - officialWebsiteUrl: undefined, + officialWebsiteUrl: softwareExternalData.websiteUrl, parentWikidataSoftware: undefined, prerogatives: { doRespectRgaa: true, @@ -96,7 +133,7 @@ describe("pgDbApi", () => { similarSoftwares: [], softwareDescription: "Super software", softwareId: expect.any(Number), - softwareName: "", + softwareName: softwareFormData.softwareName, softwareType: { os: { android: true, diff --git a/api/src/core/ports/CompileData.ts b/api/src/core/ports/CompileData.ts index a7adea17..053ed18e 100644 --- a/api/src/core/ports/CompileData.ts +++ b/api/src/core/ports/CompileData.ts @@ -1,6 +1,10 @@ import { ServiceProvider } from "../usecases/readWriteSillData"; import type { Db } from "./DbApi"; -import type { SoftwareExternalData } from "./GetSoftwareExternalData"; +import { + ParentSoftwareExternalData, + SimilarSoftwareExternalData, + SoftwareExternalData +} from "./GetSoftwareExternalData"; import type { ComptoirDuLibre } from "./ComptoirDuLibreApi"; export type CompileData = (params: { @@ -57,11 +61,8 @@ export namespace CompiledData { > & { serviceProviders: ServiceProvider[]; softwareExternalData: SoftwareExternalData | undefined; - similarExternalSoftwares: Pick< - SoftwareExternalData, - "externalId" | "label" | "description" | "isLibreSoftware" | "externalDataOrigin" - >[]; - parentWikidataSoftware: Pick | undefined; + similarExternalSoftwares: SimilarSoftwareExternalData[]; + parentWikidataSoftware: ParentSoftwareExternalData | undefined; comptoirDuLibreSoftware: | (ComptoirDuLibre.Software & { logoUrl: string | undefined; keywords: string[] | undefined }) | undefined; diff --git a/api/src/core/ports/GetSoftwareExternalData.ts b/api/src/core/ports/GetSoftwareExternalData.ts index f31680e9..26f57124 100644 --- a/api/src/core/ports/GetSoftwareExternalData.ts +++ b/api/src/core/ports/GetSoftwareExternalData.ts @@ -31,6 +31,13 @@ export type SoftwareExternalData = { license: string; }>; +export type SimilarSoftwareExternalData = Pick< + SoftwareExternalData, + "externalId" | "label" | "description" | "isLibreSoftware" | "externalDataOrigin" +>; + +export type ParentSoftwareExternalData = Pick; + export const languages = ["fr", "en"] as const; export type Language = (typeof languages)[number]; diff --git a/api/src/core/usecases/readWriteSillData/thunks.ts b/api/src/core/usecases/readWriteSillData/thunks.ts index f8d6c800..48c57fe0 100644 --- a/api/src/core/usecases/readWriteSillData/thunks.ts +++ b/api/src/core/usecases/readWriteSillData/thunks.ts @@ -139,8 +139,8 @@ export const thunks = { "comptoirDuLibreId": formData.comptoirDuLibreId, "license": formData.softwareLicense, "softwareType": formData.softwareType, - "catalogNumeriqueGouvFrId": undefined, "versionMin": formData.softwareMinimalVersion, + "catalogNumeriqueGouvFrId": undefined, "workshopUrls": [], "testUrls": [], "categories": [], diff --git a/api/src/core/usecases/readWriteSillData/types.ts b/api/src/core/usecases/readWriteSillData/types.ts index 06f56eb1..573bb89c 100644 --- a/api/src/core/usecases/readWriteSillData/types.ts +++ b/api/src/core/usecases/readWriteSillData/types.ts @@ -1,4 +1,9 @@ -import type { ExternalDataOrigin, SoftwareExternalData } from "../../ports/GetSoftwareExternalData"; +import type { + ExternalDataOrigin, + ParentSoftwareExternalData, + SimilarSoftwareExternalData, + SoftwareExternalData +} from "../../ports/GetSoftwareExternalData"; export type ServiceProvider = { name: string; @@ -54,7 +59,7 @@ export type Software = { externalId: string | undefined; externalDataOrigin: ExternalDataOrigin | undefined; softwareType: SoftwareType; - parentWikidataSoftware: Pick | undefined; + parentWikidataSoftware: ParentSoftwareExternalData | undefined; similarSoftwares: Software.SimilarSoftware[]; keywords: string[]; }; @@ -63,10 +68,7 @@ export namespace Software { export type SimilarSoftware = SimilarSoftware.ExternalSoftwareData | SimilarSoftware.Sill; export namespace SimilarSoftware { - export type ExternalSoftwareData = { isInSill: false } & Pick< - SoftwareExternalData, - "externalId" | "label" | "description" | "isLibreSoftware" | "externalDataOrigin" - >; + export type ExternalSoftwareData = { isInSill: false } & SimilarSoftwareExternalData; export type Sill = { isInSill: true; softwareName: string; softwareDescription: string }; } diff --git a/yarn.lock b/yarn.lock index a8ac54d5..acbaf5be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11842,10 +11842,10 @@ kuler@^2.0.0: resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== -kysely-ctl@^0.8.7: - version "0.8.7" - resolved "https://registry.yarnpkg.com/kysely-ctl/-/kysely-ctl-0.8.7.tgz#2eb2487b14b13e4caa52786e97ff4fedc59d7623" - integrity sha512-I69bTRzcTNh7XvHQdlPZuzCq0utNBMnWSZ3IZVI0ORpjnI0YPxeaM23Jt2OOeZCl+mnrO4ror7OjtjJNQb/OPA== +kysely-ctl@^0.8.10: + version "0.8.10" + resolved "https://registry.yarnpkg.com/kysely-ctl/-/kysely-ctl-0.8.10.tgz#799d9df83badbb88c3adc0cd4b0ab949e13376d6" + integrity sha512-uDfYhG2AJwJ1/kVRUvHewHvcvI6F8Aqnoiyx9cOzr2RZ+mip2sm5YngCJi4S/rb4/1fICIL2MuIdAGizVgpbzg== dependencies: c12 "^1.8.0" citty "^0.1.4" @@ -11857,10 +11857,10 @@ kysely-ctl@^0.8.7: std-env "^3.4.0" tsx "^4.9.0" -kysely@^0.27.3: - version "0.27.3" - resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.3.tgz#6cc6c757040500b43c4ac596cdbb12be400ee276" - integrity sha512-lG03Ru+XyOJFsjH3OMY6R/9U38IjDPfnOfDgO3ynhbDr+Dz8fak+X6L62vqu3iybQnj+lG84OttBuU9KY3L9kA== +kysely@^0.27.4: + version "0.27.4" + resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.4.tgz#96a0285467b380948b4de03b20d87e82d797449b" + integrity sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA== language-subtag-registry@^0.3.20: version "0.3.22"