Skip to content

Commit

Permalink
add compiled_softwares table
Browse files Browse the repository at this point in the history
  • Loading branch information
@JeromeBu
JeromeBu committed Jun 28, 2024
1 parent 14fe128 commit b722feb
Show file tree
Hide file tree
Showing 3 changed files with 150 additions and 11 deletions.
47 changes: 36 additions & 11 deletions api/scripts/load-git-repo-in-pg.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { z } from "zod";
import { createGitDbApi, GitDbApiParams } from "../src/core/adapters/dbApi/createGitDbApi";
import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database";
import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect";
import { CompiledData } from "../src/core/ports/CompileData";
import { Db } from "../src/core/ports/DbApi";
import SoftwareRow = Db.SoftwareRow;

Expand Down Expand Up @@ -34,13 +35,16 @@ const saveGitDbInPostgres = async ({ pgConfig, gitDbConfig }: Params) => {
});
await insertInstances({
instanceRows: instanceRows,
agentIdByEmail: agentIdByEmail,
db: pgDb
});

const compiledSoftwares = await gitDbApi.fetchCompiledData();
await insertCompiledSoftwares(compiledSoftwares, pgDb);
};

const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>) => {
console.info("Deleting than Inserting softwares");
console.info("Number of softwares to insert : ", softwareRows.length);
await db.transaction().execute(async trx => {
await trx.deleteFrom("softwares").execute();
await trx
Expand All @@ -63,9 +67,9 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>

const insertAgents = async (agentRows: Db.AgentRow[], db: Kysely<Database>) => {
console.log("Deleting than Inserting agents");
console.info("Number of agents to insert : ", agentRows.length);
await db.transaction().execute(async trx => {
await trx.deleteFrom("agents").execute();
console.log("number of agents to add : ", agentRows.length);
await trx.insertInto("agents").values(agentRows).executeTakeFirst();
});
};
Expand All @@ -86,6 +90,7 @@ const insertSoftwareReferents = async ({
db: Kysely<Database>;
}) => {
console.info("Deleting than Inserting software referents");
console.info("Number of software referents to insert : ", softwareReferentRows.length);
await db.transaction().execute(async trx => {
await trx.deleteFrom("software_referents").execute();
await trx
Expand All @@ -110,6 +115,7 @@ const insertSoftwareUsers = async ({
db: Kysely<Database>;
}) => {
console.info("Deleting than Inserting software users");
console.info("Number of software users to insert : ", softwareUserRows.length);
await db.transaction().execute(async trx => {
await trx.deleteFrom("software_users").execute();
await trx
Expand All @@ -124,16 +130,9 @@ const insertSoftwareUsers = async ({
});
};

const insertInstances = async ({
instanceRows,
agentIdByEmail,
db
}: {
instanceRows: Db.InstanceRow[];
agentIdByEmail: Record<string, number>;
db: Kysely<Database>;
}) => {
const insertInstances = async ({ instanceRows, db }: { instanceRows: Db.InstanceRow[]; db: Kysely<Database> }) => {
console.info("Deleting than Inserting instances");
console.info("Number of instances to insert : ", instanceRows.length);
await db.transaction().execute(async trx => {
await trx.deleteFrom("instances").execute();
await trx
Expand All @@ -148,6 +147,32 @@ const insertInstances = async ({
});
};

const insertCompiledSoftwares = async (
compiledSoftwares: CompiledData.Software<"private">[],
pgDb: Kysely<Database>
) => {
console.info("Deleting than Inserting compiled softwares");
console.info("Number of compiled softwares to insert : ", compiledSoftwares.length);
await pgDb.transaction().execute(async trx => {
await trx.deleteFrom("compiled_softwares").execute();
await trx
.insertInto("compiled_softwares")
.values(
compiledSoftwares.map(software => ({
softwareId: software.id,
serviceProviders: JSON.stringify(software.serviceProviders),
softwareExternalData: JSON.stringify(software.softwareExternalData),
similarExternalSoftwares: JSON.stringify(software.similarExternalSoftwares),
parentWikidataSoftware: JSON.stringify(software.parentWikidataSoftware),
comptoirDuLibreSoftware: JSON.stringify(software.comptoirDuLibreSoftware),
annuaireCnllServiceProviders: JSON.stringify(software.annuaireCnllServiceProviders),
latestVersion: JSON.stringify(software.latestVersion)
}))
)
.executeTakeFirst();
});
};

const paramsSchema: z.Schema<Params> = z.object({
pgConfig: z.object({
dbUrl: z.string()
Expand Down
95 changes: 95 additions & 0 deletions api/src/core/adapters/dbApi/kysely/kysely.database.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { Generated, JSONColumnType } from "kysely";
import type { PartialNoOptional } from "../../../../tools/PartialNoOptional";

export type Database = {
agents: AgentsTable;
software_referents: SoftwareReferentsTable;
software_users: SoftwareUsersTable;
instances: InstancesTable;
softwares: SoftwaresTable;
compiled_softwares: CompiledSoftwaresTable;
};

type AgentsTable = {
Expand Down Expand Up @@ -92,3 +94,96 @@ type SoftwaresTable = {
logoUrl: string | null;
keywords: JSONColumnType<string[]>;
};

// ---------- compiled data ----------

type ComptoirDuLibreProvider = {
id: number;
url: string;
name: string;
type: string;
external_resources: {
website: string | null;
};
};

type ComptoirDuLibreUser = {
id: number;
url: string;
name: string;
type: string;
external_resources: {
website: string | null;
};
};

type ComptoirDuLibreSoftware = {
softwareId: number;
comptoirDuLibreId: number;
logoUrl: string | undefined;
keywords: string[] | undefined;
created: string;
modified: string;
url: string;
name: string;
licence: string;
external_resources: {
website: string | null;
repository: string | null;
};
providers: ComptoirDuLibreProvider[];
users: ComptoirDuLibreUser[];
};

type ServiceProvider = {
name: string;
website?: string;
cdlUrl?: string;
cnllUrl?: string;
siren?: string;
};

type ExternalDataDeveloper = {
name: string;
id: string;
};

type LocalizedString = string | Partial<Record<string, string>>;

type SoftwareExternalData = {
externalId: string;
externalDataOrigin: "wikidata" | "HAL";
developers: ExternalDataDeveloper[];
label: LocalizedString;
description: LocalizedString;
isLibreSoftware: boolean;
} & PartialNoOptional<{
logoUrl: string;
framaLibreId: string;
websiteUrl: string;
sourceUrl: string;
documentationUrl: string;
license: string;
}>;

type CompiledSoftwaresTable = {
softwareId: number;
serviceProviders: JSONColumnType<ServiceProvider[]>;
softwareExternalData: JSONColumnType<SoftwareExternalData> | null;
similarExternalSoftwares: JSONColumnType<
Pick<SoftwareExternalData, "externalId" | "label" | "description" | "isLibreSoftware" | "externalDataOrigin">[]
>;
parentWikidataSoftware: JSONColumnType<Pick<SoftwareExternalData, "externalId" | "label" | "description">> | null;
comptoirDuLibreSoftware: JSONColumnType<ComptoirDuLibreSoftware> | null;
annuaireCnllServiceProviders: JSONColumnType<
{
name: string;
siren: string;
url: string;
}[]
> | null;
latestVersion: JSONColumnType<{
semVer: string;
publicationTime: number;
}> | null;
};
19 changes: 19 additions & 0 deletions api/src/core/adapters/dbApi/kysely/migrations/1719576701920_add-compiled-tables.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import type { Kysely } from "kysely";

export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.createTable("compiled_softwares")
.addColumn("softwareId", "integer", col => col.notNull().references("softwares.id").onDelete("cascade"))
.addColumn("serviceProviders", "jsonb", col => col.notNull())
.addColumn("softwareExternalData", "jsonb")
.addColumn("similarExternalSoftwares", "jsonb", col => col.notNull())
.addColumn("parentWikidataSoftware", "jsonb")
.addColumn("comptoirDuLibreSoftware", "jsonb")
.addColumn("annuaireCnllServiceProviders", "jsonb")
.addColumn("latestVersion", "jsonb")
.execute();
}

export async function down(db: Kysely<any>): Promise<void> {
await db.schema.dropTable("compiled_softwares").execute();
}

0 comments on commit b722feb

Please sign in to comment.