diff --git a/api/scripts/load-git-repo-in-pg.ts b/api/scripts/load-git-repo-in-pg.ts index d1486f7f..79069476 100644 --- a/api/scripts/load-git-repo-in-pg.ts +++ b/api/scripts/load-git-repo-in-pg.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { createGitDbApi, GitDbApiParams } from "../src/core/adapters/dbApi/createGitDbApi"; import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database"; import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect"; +import { CompiledData } from "../src/core/ports/CompileData"; import { Db } from "../src/core/ports/DbApi"; import SoftwareRow = Db.SoftwareRow; @@ -34,13 +35,16 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => { }); await insertInstances({ instanceRows: instanceRows, - agentIdByEmail: agentIdByEmail, db: pgDb }); + + const compiledSoftwares = await gitDbApi.fetchCompiledData(); + await insertCompiledSoftwares(compiledSoftwares, pgDb); }; const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely) => { console.info("Deleting than Inserting softwares"); + console.info("Number of softwares to insert : ", softwareRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("softwares").execute(); await trx @@ -63,9 +67,9 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely const insertAgents = async (agentRows: Db.AgentRow[], db: Kysely) => { console.log("Deleting than Inserting agents"); + console.info("Number of agents to insert : ", agentRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("agents").execute(); - console.log("number of agents to add : ", agentRows.length); await trx.insertInto("agents").values(agentRows).executeTakeFirst(); }); }; @@ -86,6 +90,7 @@ const insertSoftwareReferents = async ({ db: Kysely; }) => { console.info("Deleting than Inserting software referents"); + console.info("Number of software referents to insert : ", softwareReferentRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("software_referents").execute(); await trx @@ -110,6 +115,7 @@ const insertSoftwareUsers = async ({ db: Kysely; }) => { console.info("Deleting than Inserting software users"); + console.info("Number of software users to insert : ", softwareUserRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("software_users").execute(); await trx @@ -124,16 +130,9 @@ const insertSoftwareUsers = async ({ }); }; -const insertInstances = async ({ - instanceRows, - agentIdByEmail, - db -}: { - instanceRows: Db.InstanceRow[]; - agentIdByEmail: Record; - db: Kysely; -}) => { +const insertInstances = async ({ instanceRows, db }: { instanceRows: Db.InstanceRow[]; db: Kysely }) => { console.info("Deleting than Inserting instances"); + console.info("Number of instances to insert : ", instanceRows.length); await db.transaction().execute(async trx => { await trx.deleteFrom("instances").execute(); await trx @@ -148,6 +147,32 @@ const insertInstances = async ({ }); }; +const insertCompiledSoftwares = async ( + compiledSoftwares: CompiledData.Software<"private">[], + pgDb: Kysely +) => { + console.info("Deleting than Inserting compiled softwares"); + console.info("Number of compiled softwares to insert : ", compiledSoftwares.length); + await pgDb.transaction().execute(async trx => { + await trx.deleteFrom("compiled_softwares").execute(); + await trx + .insertInto("compiled_softwares") + .values( + compiledSoftwares.map(software => ({ + softwareId: software.id, + serviceProviders: JSON.stringify(software.serviceProviders), + softwareExternalData: JSON.stringify(software.softwareExternalData), + similarExternalSoftwares: JSON.stringify(software.similarExternalSoftwares), + parentWikidataSoftware: JSON.stringify(software.parentWikidataSoftware), + comptoirDuLibreSoftware: JSON.stringify(software.comptoirDuLibreSoftware), + annuaireCnllServiceProviders: JSON.stringify(software.annuaireCnllServiceProviders), + latestVersion: JSON.stringify(software.latestVersion) + })) + ) + .executeTakeFirst(); + }); +}; + const paramsSchema: z.Schema = z.object({ pgConfig: z.object({ dbUrl: z.string() diff --git a/api/src/core/adapters/dbApi/kysely/kysely.database.ts b/api/src/core/adapters/dbApi/kysely/kysely.database.ts index f5f7c13f..5cf69f05 100644 --- a/api/src/core/adapters/dbApi/kysely/kysely.database.ts +++ b/api/src/core/adapters/dbApi/kysely/kysely.database.ts @@ -1,4 +1,5 @@ import { Generated, JSONColumnType } from "kysely"; +import type { PartialNoOptional } from "../../../../tools/PartialNoOptional"; export type Database = { agents: AgentsTable; @@ -6,6 +7,7 @@ export type Database = { software_users: SoftwareUsersTable; instances: InstancesTable; softwares: SoftwaresTable; + compiled_softwares: CompiledSoftwaresTable; }; type AgentsTable = { @@ -92,3 +94,96 @@ type SoftwaresTable = { logoUrl: string | null; keywords: JSONColumnType; }; + +// ---------- compiled data ---------- + +type ComptoirDuLibreProvider = { + id: number; + url: string; + name: string; + type: string; + external_resources: { + website: string | null; + }; +}; + +type ComptoirDuLibreUser = { + id: number; + url: string; + name: string; + type: string; + external_resources: { + website: string | null; + }; +}; + +type ComptoirDuLibreSoftware = { + softwareId: number; + comptoirDuLibreId: number; + logoUrl: string | undefined; + keywords: string[] | undefined; + created: string; + modified: string; + url: string; + name: string; + licence: string; + external_resources: { + website: string | null; + repository: string | null; + }; + providers: ComptoirDuLibreProvider[]; + users: ComptoirDuLibreUser[]; +}; + +type ServiceProvider = { + name: string; + website?: string; + cdlUrl?: string; + cnllUrl?: string; + siren?: string; +}; + +type ExternalDataDeveloper = { + name: string; + id: string; +}; + +type LocalizedString = string | Partial>; + +type SoftwareExternalData = { + externalId: string; + externalDataOrigin: "wikidata" | "HAL"; + developers: ExternalDataDeveloper[]; + label: LocalizedString; + description: LocalizedString; + isLibreSoftware: boolean; +} & PartialNoOptional<{ + logoUrl: string; + framaLibreId: string; + websiteUrl: string; + sourceUrl: string; + documentationUrl: string; + license: string; +}>; + +type CompiledSoftwaresTable = { + softwareId: number; + serviceProviders: JSONColumnType; + softwareExternalData: JSONColumnType | null; + similarExternalSoftwares: JSONColumnType< + Pick[] + >; + parentWikidataSoftware: JSONColumnType> | null; + comptoirDuLibreSoftware: JSONColumnType | null; + annuaireCnllServiceProviders: JSONColumnType< + { + name: string; + siren: string; + url: string; + }[] + > | null; + latestVersion: JSONColumnType<{ + semVer: string; + publicationTime: number; + }> | null; +}; diff --git a/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts b/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts new file mode 100644 index 00000000..7ad163a7 --- /dev/null +++ b/api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts @@ -0,0 +1,19 @@ +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable("compiled_softwares") + .addColumn("softwareId", "integer", col => col.notNull().references("softwares.id").onDelete("cascade")) + .addColumn("serviceProviders", "jsonb", col => col.notNull()) + .addColumn("softwareExternalData", "jsonb") + .addColumn("similarExternalSoftwares", "jsonb", col => col.notNull()) + .addColumn("parentWikidataSoftware", "jsonb") + .addColumn("comptoirDuLibreSoftware", "jsonb") + .addColumn("annuaireCnllServiceProviders", "jsonb") + .addColumn("latestVersion", "jsonb") + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable("compiled_softwares").execute(); +}