implement postgres adapter

.github/workflows/ci.yaml

@@ -10,6 +10,17 @@ on:
 jobs:
   validations:
     runs-on: ubuntu-latest
+    env:
+      DATABASE_URL: postgresql://sill:pg_password@localhost:5432/sill
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_USER: sill
+          POSTGRES_PASSWORD: pg_password
+          POSTGRES_DB: sill
+        ports:
+          - 5432:5432
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -18,6 +29,8 @@ jobs:
       - uses: bahmutov/npm-install@v1
       - name: Build back
         run: cd api && yarn build
+      - name: Migrate db
+        run: cd api && yarn migrate latest
       - name: Fullcheck
         run: yarn fullcheck
   #

Dockerfile

@@ -23,7 +23,7 @@ RUN yarn build
 
 WORKDIR /app/api
 RUN rm -r src
-RUN cp dist -r src/
+RUN cp dist/src -r src/
 RUN npx ncc build src/main.js
 
 WORKDIR /app
@@ -39,7 +39,7 @@ RUN apk add --no-cache \
   git \
   openssh-client \
   ca-certificates
-COPY --from=build /app/api/dist/index.js .
+COPY --from=build /app/api/dist/src/lib/index.js .
 # For reading the version number
 COPY --from=build /app/package.json .
 ENTRYPOINT sh -c "forever index.js"

api/.config/kysely.config.ts

@@ -0,0 +1,16 @@
+import { PostgresDialect } from "kysely";
+import { defineConfig } from "kysely-ctl";
+import { Pool } from "pg";
+
+const dialect = new PostgresDialect({
+    pool: new Pool({
+        connectionString: process.env.DATABASE_URL
+    })
+});
+
+export default defineConfig({
+    dialect,
+    migrations: {
+        migrationFolder: "src/core/adapters/dbApi/kysely/migrations"
+    }
+});

api/package.json

@@ -4,22 +4,24 @@
     "description": "The backend of code.gouv.fr/sill",
     "repository": {
         "type": "git",
-        "url": "git://github.com/codegouvfr/sill-api.git"
+        "url": "git://github.com/codegouvfr/sill.git"
     },
-    "main": "dist/lib/index.js",
-    "types": "dist/lib/index.d.ts",
+    "main": "dist/src/lib/index.js",
+    "types": "dist/src/lib/index.d.ts",
     "scripts": {
+        "migrate": "dotenv -e ../.env -- kysely migrate",
         "prepare": "[ ! -f .env.local.sh ] && cp .env.sh .env.local.sh || true",
         "test": "vitest --watch=false",
         "dev": "yarn build && yarn start",
         "build": "tsc",
-        "start": "dotenv -e ../.env -- forever dist/main.js",
+        "start": "dotenv -e ../.env -- forever dist/src/main.js",
         "_format": "prettier \"**/*.{ts,tsx,json,md}\"",
         "format": "yarn run _format --write",
         "format:check": "yarn run _format --list-different",
         "link-in-web": "ts-node --skipProject scripts/link-in-app.ts sill-web",
         "reset-data-test": "ts-node --skipProject scripts/reset-data-test.ts",
         "compile-data": "./.env.local.sh ts-node --skipProject scripts/compile-data.ts",
+        "load-git-repo-in-pg": "dotenv -e ../.env -- ts-node --skipProject scripts/load-git-repo-in-pg.ts",
         "typecheck": "tsc --noEmit"
     },
     "author": "DINUM",
@@ -30,7 +32,7 @@
         "!dist/tsconfig.tsbuildinfo"
     ],
     "keywords": [],
-    "homepage": "https://github.com/codegouvfr/sill-api",
+    "homepage": "https://github.com/codegouvfr/sill",
     "devDependencies": {
         "@octokit/rest": "^18.12.0",
         "@types/compression": "^1.7.2",
@@ -74,7 +76,11 @@
         "@octokit/graphql": "^7.0.2",
         "@retorquere/bibtex-parser": "^7.0.11",
         "@trpc/server": "^10.18.0",
+        "@types/pg": "^8.11.6",
         "jwt-decode": "^3.1.2",
+        "kysely": "^0.27.4",
+        "kysely-ctl": "^0.8.10",
+        "pg": "^8.11.5",
         "semver": "^7.5.4",
         "tsafe": "^1.6.6",
         "zod": "^3.21.4"

api/scripts/compile-data.ts

@@ -4,7 +4,8 @@ import { env } from "../src/env";
 (async () => {
     const { core } = await bootstrapCore({
         "keycloakUserApiParams": undefined,
-        "gitDbApiParams": {
+        "dbConfig": {
+            "dbKind": "git",
             "dataRepoSshUrl": "[email protected]:codegouvfr/sill-data.git",
             "sshPrivateKey": env.sshPrivateKeyForGit,
             "sshPrivateKeyName": env.sshPrivateKeyForGitName

api/scripts/load-git-repo-in-pg.ts

@@ -0,0 +1,265 @@
+import { InsertObject, Kysely } from "kysely";
+import { z } from "zod";
+import { createGitDbApi, GitDbApiParams } from "../src/core/adapters/dbApi/createGitDbApi";
+import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database";
+import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect";
+import { CompiledData } from "../src/core/ports/CompileData";
+import { Db } from "../src/core/ports/DbApi";
+import { ExternalDataOrigin } from "../src/core/ports/GetSoftwareExternalData";
+import SoftwareRow = Db.SoftwareRow;
+
+export type Params = {
+    pgConfig: { dbUrl: string };
+    gitDbConfig: GitDbApiParams;
+};
+
+const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => {
+    const { dbApi: gitDbApi } = createGitDbApi(gitDbConfig);
+    if (!pgConfig.dbUrl) throw new Error("Missing PG database url, please set the DATABASE_URL environnement variable");
+    const pgDb = new Kysely<Database>({ dialect: createPgDialect(pgConfig.dbUrl) });
+
+    const { softwareRows, agentRows, softwareReferentRows, softwareUserRows, instanceRows } = await gitDbApi.fetchDb();
+
+    await insertSoftwares(softwareRows, pgDb);
+    await insertAgents(agentRows, pgDb);
+
+    const agentIdByEmail = await makeGetAgentIdByEmail(pgDb);
+    await insertSoftwareReferents({
+        softwareReferentRows: softwareReferentRows,
+        agentIdByEmail: agentIdByEmail,
+        db: pgDb
+    });
+    await insertSoftwareUsers({
+        softwareUserRows: softwareUserRows,
+        agentIdByEmail: agentIdByEmail,
+        db: pgDb
+    });
+    await insertInstances({
+        instanceRows: instanceRows,
+        db: pgDb
+    });
+
+    const compiledSoftwares = await gitDbApi.fetchCompiledData();
+    await insertCompiledSoftwaresAndSoftwareExternalData(compiledSoftwares, pgDb);
+};
+
+const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>) => {
+    console.info("Deleting than Inserting softwares");
+    console.info("Number of softwares to insert : ", softwareRows.length);
+    await db.transaction().execute(async trx => {
+        await trx.deleteFrom("softwares").execute();
+        await trx.deleteFrom("softwares__similar_software_external_datas").execute();
+        await trx
+            .insertInto("softwares")
+            .values(
+                softwareRows.map(({ similarSoftwareExternalDataIds: _, ...row }) => ({
+                    ...row,
+                    dereferencing: row.dereferencing ? JSON.stringify(row.dereferencing) : null,
+                    softwareType: JSON.stringify(row.softwareType),
+                    workshopUrls: JSON.stringify(row.workshopUrls),
+                    testUrls: JSON.stringify(row.testUrls),
+                    categories: JSON.stringify(row.categories),
+                    keywords: JSON.stringify(row.keywords)
+                }))
+            )
+            .executeTakeFirst();
+
+        await trx
+            .insertInto("softwares__similar_software_external_datas")
+            .values(
+                softwareRows.flatMap(row =>
+                    Array.from(new Set(row.similarSoftwareExternalDataIds)).map(externalId => ({
+                        softwareId: row.id,
+                        similarExternalId: externalId
+                    }))
+                )
+            )
+            .execute();
+    });
+};
+
+const insertAgents = async (agentRows: Db.AgentRow[], db: Kysely<Database>) => {
+    console.log("Deleting than Inserting agents");
+    console.info("Number of agents to insert : ", agentRows.length);
+    await db.transaction().execute(async trx => {
+        await trx.deleteFrom("agents").execute();
+        await trx.insertInto("agents").values(agentRows).executeTakeFirst();
+    });
+};
+
+const makeGetAgentIdByEmail = async (db: Kysely<Database>): Promise<Record<string, number>> => {
+    console.info("Fetching agents, to map email to id");
+    const agents = await db.selectFrom("agents").select(["email", "id"]).execute();
+    return agents.reduce((acc, agent) => ({ ...acc, [agent.email]: agent.id }), {});
+};
+
+const insertSoftwareReferents = async ({
+    softwareReferentRows,
+    agentIdByEmail,
+    db
+}: {
+    softwareReferentRows: Db.SoftwareReferentRow[];
+    agentIdByEmail: Record<string, number>;
+    db: Kysely<Database>;
+}) => {
+    console.info("Deleting than Inserting software referents");
+    console.info("Number of software referents to insert : ", softwareReferentRows.length);
+    await db.transaction().execute(async trx => {
+        await trx.deleteFrom("software_referents").execute();
+        await trx
+            .insertInto("software_referents")
+            .values(
+                softwareReferentRows.map(({ agentEmail, ...rest }) => ({
+                    ...rest,
+                    agentId: agentIdByEmail[agentEmail]
+                }))
+            )
+            .executeTakeFirst();
+    });
+};
+
+const insertSoftwareUsers = async ({
+    softwareUserRows,
+    agentIdByEmail,
+    db
+}: {
+    softwareUserRows: Db.SoftwareUserRow[];
+    agentIdByEmail: Record<string, number>;
+    db: Kysely<Database>;
+}) => {
+    console.info("Deleting than Inserting software users");
+    console.info("Number of software users to insert : ", softwareUserRows.length);
+    await db.transaction().execute(async trx => {
+        await trx.deleteFrom("software_users").execute();
+        await trx
+            .insertInto("software_users")
+            .values(
+                softwareUserRows.map(({ agentEmail, ...rest }) => ({
+                    ...rest,
+                    agentId: agentIdByEmail[agentEmail]
+                }))
+            )
+            .executeTakeFirst();
+    });
+};
+
+const insertInstances = async ({ instanceRows, db }: { instanceRows: Db.InstanceRow[]; db: Kysely<Database> }) => {
+    console.info("Deleting than Inserting instances");
+    console.info("Number of instances to insert : ", instanceRows.length);
+    await db.transaction().execute(async trx => {
+        await trx.deleteFrom("instances").execute();
+        await trx.insertInto("instances").values(instanceRows).executeTakeFirst();
+    });
+};
+
+const insertCompiledSoftwaresAndSoftwareExternalData = async (
+    compiledSoftwares: CompiledData.Software<"private">[],
+    pgDb: Kysely<Database>
+) => {
+    console.info("Deleting than Inserting compiled softwares");
+    console.info("Number of compiled softwares to insert : ", compiledSoftwares.length);
+    await pgDb.transaction().execute(async trx => {
+        await trx.deleteFrom("compiled_softwares").execute();
+        await trx
+            .insertInto("compiled_softwares")
+            .values(
+                compiledSoftwares.map(
+                    (software): InsertObject<Database, "compiled_softwares"> => ({
+                        softwareId: software.id,
+                        serviceProviders: JSON.stringify(software.serviceProviders),
+                        comptoirDuLibreSoftware: JSON.stringify(software.comptoirDuLibreSoftware),
+                        annuaireCnllServiceProviders: JSON.stringify(software.annuaireCnllServiceProviders),
+                        latestVersion: JSON.stringify(software.latestVersion)
+                    })
+                )
+            )
+            .executeTakeFirst();
+
+        await trx.deleteFrom("software_external_datas").execute();
+
+        await trx
+            .insertInto("software_external_datas")
+            .values(
+                compiledSoftwares
+                    .filter(
+                        (
+                            software
+                        ): software is CompiledData.Software.Private & {
+                            softwareExternalData: {
+                                externalId: string;
+                                externalDataOrigin: ExternalDataOrigin;
+                            };
+                        } =>
+                            software.softwareExternalData?.externalId !== undefined &&
+                            software.softwareExternalData?.externalDataOrigin !== undefined
+                    )
+                    .map(
+                        ({ softwareExternalData }): InsertObject<Database, "software_external_datas"> => ({
+                            externalId: softwareExternalData.externalId,
+                            externalDataOrigin: softwareExternalData.externalDataOrigin,
+                            developers: JSON.stringify(softwareExternalData?.developers ?? []),
+                            label: JSON.stringify(softwareExternalData?.label ?? {}),
+                            description: JSON.stringify(softwareExternalData?.description ?? {}),
+                            isLibreSoftware: softwareExternalData?.isLibreSoftware ?? false,
+                            logoUrl: softwareExternalData?.logoUrl ?? null,
+                            framaLibreId: softwareExternalData?.framaLibreId ?? null,
+                            websiteUrl: softwareExternalData?.websiteUrl ?? null,
+                            sourceUrl: softwareExternalData?.sourceUrl ?? null,
+                            documentationUrl: softwareExternalData?.documentationUrl ?? null,
+                            license: softwareExternalData?.license ?? null
+                        })
+                    )
+            )
+            .onConflict(conflict => conflict.column("externalId").doNothing())
+            .executeTakeFirst();
+
+        await trx
+            .insertInto("software_external_datas")
+            .values(
+                compiledSoftwares
+                    .filter(s => s.similarExternalSoftwares.length > 0)
+                    .flatMap(s =>
+                        (s.similarExternalSoftwares ?? []).map(similarExternalSoftware => ({
+                            externalId: similarExternalSoftware.externalId,
+                            externalDataOrigin: similarExternalSoftware.externalDataOrigin,
+                            developers: JSON.stringify([]),
+                            label: JSON.stringify(similarExternalSoftware?.label ?? {}),
+                            description: JSON.stringify(similarExternalSoftware?.description ?? {}),
+                            isLibreSoftware: similarExternalSoftware?.isLibreSoftware ?? false
+                        }))
+                    )
+            )
+            .onConflict(conflict => conflict.column("externalId").doNothing())
+            .executeTakeFirst();
+    });
+};
+
+const paramsSchema: z.Schema<Params> = z.object({
+    pgConfig: z.object({
+        dbUrl: z.string()
+    }),
+    gitDbConfig: z.object({
+        dataRepoSshUrl: z.string(),
+        sshPrivateKey: z.string(),
+        sshPrivateKeyName: z.string()
+    })
+});
+
+const timerName = "Script duration";
+console.time(timerName);
+
+saveGitDbInPostgres(
+    paramsSchema.parse({
+        pgConfig: { dbUrl: process.env.DATABASE_URL },
+        gitDbConfig: {
+            dataRepoSshUrl: process.env.SILL_DATA_REPO_SSH_URL,
+            sshPrivateKey: process.env.SILL_SSH_PRIVATE_KEY,
+            sshPrivateKeyName: process.env.SILL_SSH_NAME
+        }
+    })
+)
+    .then(() => {
+        console.log("Load git db in postgres with success");
+        process.exit(0);
+    })
+    .finally(() => console.timeEnd(timerName));