Skip to content

Commit

Permalink
improve query to get all private compile software
Browse files Browse the repository at this point in the history
  • Loading branch information
JeromeBu committed Jul 21, 2024
1 parent 398c02c commit 275aaa4
Show file tree
Hide file tree
Showing 7 changed files with 168 additions and 96 deletions.
36 changes: 34 additions & 2 deletions api/scripts/load-git-repo-in-pg.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,13 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>
console.info("Number of softwares to insert : ", softwareRows.length);
await db.transaction().execute(async trx => {
await trx.deleteFrom("softwares").execute();
await trx.deleteFrom("softwares__similar_software_external_datas").execute();
await trx
.insertInto("softwares")
.values(
softwareRows.map(row => ({
softwareRows.map(({ similarSoftwareExternalDataIds: _, ...row }) => ({
...row,
dereferencing: row.dereferencing ? JSON.stringify(row.dereferencing) : null,
similarSoftwareExternalDataIds: JSON.stringify(row.similarSoftwareExternalDataIds),
softwareType: JSON.stringify(row.softwareType),
workshopUrls: JSON.stringify(row.workshopUrls),
testUrls: JSON.stringify(row.testUrls),
Expand All @@ -63,6 +63,18 @@ const insertSoftwares = async (softwareRows: SoftwareRow[], db: Kysely<Database>
}))
)
.executeTakeFirst();

await trx
.insertInto("softwares__similar_software_external_datas")
.values(
softwareRows.flatMap(row =>
Array.from(new Set(row.similarSoftwareExternalDataIds)).map(externalId => ({
softwareId: row.id,
similarExternalId: externalId
}))
)
)
.execute();
});
};

Expand Down Expand Up @@ -168,6 +180,7 @@ const insertCompiledSoftwaresAndSoftwareExternalData = async (
.executeTakeFirst();

await trx.deleteFrom("software_external_datas").execute();

await trx
.insertInto("software_external_datas")
.values(
Expand Down Expand Up @@ -203,6 +216,25 @@ const insertCompiledSoftwaresAndSoftwareExternalData = async (
)
.onConflict(conflict => conflict.column("externalId").doNothing())
.executeTakeFirst();

await trx
.insertInto("software_external_datas")
.values(
compiledSoftwares
.filter(s => s.similarExternalSoftwares.length > 0)
.flatMap(s =>
(s.similarExternalSoftwares ?? []).map(similarExternalSoftware => ({
externalId: similarExternalSoftware.externalId,
externalDataOrigin: similarExternalSoftware.externalDataOrigin,
developers: JSON.stringify([]),
label: JSON.stringify(similarExternalSoftware?.label ?? {}),
description: JSON.stringify(similarExternalSoftware?.description ?? {}),
isLibreSoftware: similarExternalSoftware?.isLibreSoftware ?? false
}))
)
)
.onConflict(conflict => conflict.column("externalId").doNothing())
.executeTakeFirst();
});
};

Expand Down
66 changes: 36 additions & 30 deletions api/src/core/adapters/dbApi/kysely/createGetCompiledData.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { CompiledData } from "../../../ports/CompileData";
import { Db } from "../../../ports/DbApi";
import { ParentSoftwareExternalData, SoftwareExternalData } from "../../../ports/GetSoftwareExternalData";
import { Database } from "./kysely.database";
import { convertNullValuesToUndefined, isNotNull, jsonBuildObject } from "./kysely.utils";
import { convertNullValuesToUndefined, isNotNull, jsonBuildObject, jsonStripNulls } from "./kysely.utils";

export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise<CompiledData<"private">> => {
console.time("agentById query");
Expand Down Expand Up @@ -90,35 +90,26 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
.case()
.when("ext.externalId", "is not", null)
.then(
jsonBuildObject({
externalId: ref("ext.externalId"),
externalDataOrigin: ref("ext.externalDataOrigin"),
developers: ref("ext.developers"),
label: ref("ext.label"),
description: ref("ext.description"),
isLibreSoftware: ref("ext.isLibreSoftware"),
logoUrl: ref("ext.logoUrl"),
framaLibreId: ref("ext.framaLibreId"),
websiteUrl: ref("ext.websiteUrl"),
sourceUrl: ref("ext.sourceUrl"),
documentationUrl: ref("ext.documentationUrl")
}).$castTo<SoftwareExternalData>()
jsonStripNulls(
jsonBuildObject({
externalId: ref("ext.externalId"),
externalDataOrigin: ref("ext.externalDataOrigin"),
developers: ref("ext.developers"),
label: ref("ext.label"),
description: ref("ext.description"),
isLibreSoftware: ref("ext.isLibreSoftware"),
logoUrl: ref("ext.logoUrl"),
framaLibreId: ref("ext.framaLibreId"),
websiteUrl: ref("ext.websiteUrl"),
sourceUrl: ref("ext.sourceUrl"),
documentationUrl: ref("ext.documentationUrl"),
license: ref("ext.license")
})
).$castTo<SoftwareExternalData>()
)
.end()
.as("softwareExternalData"),
({ ref, fn }) =>
fn
.jsonAgg(
jsonBuildObject({
externalId: ref("similarExt.externalId"),
label: ref("similarExt.label"),
description: ref("similarExt.description"),
isLibreSoftware: ref("similarExt.isLibreSoftware"),
externalDataOrigin: ref("similarExt.externalDataOrigin")
}).$castTo<SoftwareExternalData>()
)
.filterWhere("similarExt.externalId", "is not", null)
.as("similarExternalSoftwares"),
({ fn }) => fn.jsonAgg("similarExt").distinct().as("similarExternalSoftwares"),
({ fn }) => fn.jsonAgg("users").distinct().as("users"),
({ fn }) => fn.jsonAgg("referents").distinct().as("referents"),
({ fn }) => fn.jsonAgg("instances").distinct().as("instances")
Expand Down Expand Up @@ -158,7 +149,16 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
parentWikidataSoftware: parentWikidataSoftware ?? undefined,
dereferencing: dereferencing ?? undefined,
serviceProviders: serviceProviders ?? [],
similarExternalSoftwares: similarExternalSoftwares ?? [],
similarExternalSoftwares: (similarExternalSoftwares ?? [])
.filter(isNotNull)
.map(similar => ({
"externalId": similar.externalId!,
"externalDataOrigin": similar.externalDataOrigin!,
"label": similar.label!,
"description": similar.description!,
"isLibreSoftware": similar.isLibreSoftware!
}))
.sort((a, b) => a.externalId.localeCompare(b.externalId)),
users: users.filter(isNotNull).map(user => ({
...(user as any),
organization: agentById[user.agentId!]?.organization
Expand All @@ -167,8 +167,13 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
...(referent as any),
organization: agentById[referent.agentId!]?.organization
})),
instances: instances.filter(isNotNull).map(instance => ({
...(instance as any)
instances: (instances ?? []).filter(isNotNull).map(instance => ({
id: instance.id!,
organization: instance.organization!,
targetAudience: instance.targetAudience!,
publicUrl: instance.publicUrl ?? undefined,
addedByAgentEmail: instance.addedByAgentEmail!,
otherWikidataSoftwares: []
}))
};
}
Expand All @@ -177,5 +182,6 @@ export const createGetCompiledData = (db: Kysely<Database>) => async (): Promise
return processedSoftwares;
});

console.log("numberOfCompiledSoftwares : ", compliedSoftwares.length);
return compliedSoftwares;
};
71 changes: 38 additions & 33 deletions api/src/core/adapters/dbApi/kysely/createPgSoftwareRepository.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,36 +29,44 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
assert<Equals<typeof rest, {}>>();

const now = Date.now();
await db
.insertInto("softwares")
.values({
name: softwareName,
description: softwareDescription,
license: softwareLicense,
logoUrl: softwareLogoUrl,
versionMin: softwareMinimalVersion,
referencedSinceTime: now,
updateTime: now,
dereferencing: undefined,
isStillInObservation: false,
parentSoftwareWikidataId: undefined,
doRespectRgaa: doRespectRgaa,
isFromFrenchPublicService: isFromFrenchPublicService,
isPresentInSupportContract: isPresentInSupportContract,
similarSoftwareExternalDataIds: JSON.stringify(similarSoftwareExternalDataIds),
externalId: externalId,
externalDataOrigin: externalDataOrigin,
comptoirDuLibreId: comptoirDuLibreId,
softwareType: JSON.stringify(softwareType),
catalogNumeriqueGouvFrId: undefined,
workshopUrls: JSON.stringify([]),
testUrls: JSON.stringify([]),
categories: JSON.stringify([]),
generalInfoMd: undefined,
addedByAgentEmail: agentEmail,
keywords: JSON.stringify(softwareKeywords)
})
.execute();

await db.transaction().execute(async trx => {
const { softwareId } = await trx
.insertInto("softwares")
.values({
name: softwareName,
description: softwareDescription,
license: softwareLicense,
logoUrl: softwareLogoUrl,
versionMin: softwareMinimalVersion,
referencedSinceTime: now,
updateTime: now,
dereferencing: undefined,
isStillInObservation: false,
parentSoftwareWikidataId: undefined,
doRespectRgaa: doRespectRgaa,
isFromFrenchPublicService: isFromFrenchPublicService,
isPresentInSupportContract: isPresentInSupportContract,
externalId: externalId,
externalDataOrigin: externalDataOrigin,
comptoirDuLibreId: comptoirDuLibreId,
softwareType: JSON.stringify(softwareType),
catalogNumeriqueGouvFrId: undefined,
workshopUrls: JSON.stringify([]),
testUrls: JSON.stringify([]),
categories: JSON.stringify([]),
generalInfoMd: undefined,
addedByAgentEmail: agentEmail,
keywords: JSON.stringify(softwareKeywords)
})
.returning("id as softwareId")
.executeTakeFirstOrThrow();

await trx
.insertInto("softwares__similar_software_external_datas")
.values(similarSoftwareExternalDataIds.map(similarExternalId => ({ softwareId, similarExternalId })))
.execute();
});
},
update: async ({ formData, softwareSillId, agentEmail }) => {
const {
Expand Down Expand Up @@ -95,7 +103,6 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
doRespectRgaa: doRespectRgaa,
isFromFrenchPublicService: isFromFrenchPublicService,
isPresentInSupportContract: isPresentInSupportContract,
similarSoftwareExternalDataIds: JSON.stringify(similarSoftwareExternalDataIds),
externalId: externalId,
comptoirDuLibreId: comptoirDuLibreId,
softwareType: JSON.stringify(softwareType),
Expand Down Expand Up @@ -176,7 +183,6 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
.else(null)
.end()
.as("parentExternalData"),
"s.similarSoftwareExternalDataIds",
"s.keywords",

({ ref }) =>
Expand Down Expand Up @@ -218,7 +224,6 @@ export const createPgSoftwareRepository = (db: Kysely<Database>): SoftwareReposi
({
testUrls,
serviceProviders,
similarSoftwareExternalDataIds,
parentExternalData,
updateTime,
addedTime,
Expand Down
1 change: 0 additions & 1 deletion api/src/core/adapters/dbApi/kysely/kysely.database.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ type SoftwaresTable = {
doRespectRgaa: boolean | null;
isFromFrenchPublicService: boolean;
isPresentInSupportContract: boolean;
similarSoftwareExternalDataIds: JSONColumnType<string[]>;
parentSoftwareWikidataId: string | null;
externalId: string | null;
externalDataOrigin: "wikidata" | "HAL" | null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ export async function up(db: Kysely<any>): Promise<void> {
.addColumn("doRespectRgaa", "boolean")
// from ???
.addColumn("isStillInObservation", "boolean", col => col.notNull())
.addColumn("similarSoftwareExternalDataIds", "jsonb")
.addColumn("parentSoftwareWikidataId", "text")
.addColumn("catalogNumeriqueGouvFrId", "text")
.addColumn("workshopUrls", "jsonb", col => col.notNull())
Expand Down
Original file line number Diff line number Diff line change
@@ -1,46 +1,54 @@
import type { Kysely } from "kysely";

const softwares_externalIdIdx = "softwares__externalId_idx";
const softwares_parentExternalIdIdx = "softwares__parentExternalId_idx";
const compiledSoftwares_SoftwareIdIdx = "compiled_softwares__softwareId_idx";
const compiledSoftwares_GroupByIdx = "compiled_softwares_group_by_idx";
const software_similarExternalIdIdx = "softwares_similarExternalId_idx";
const software_softwareIdIdx = "softwares_similarSoftwareId_idx";
const softwareReferents_softwareIdIdx = "softwareReferents_software_idx";
const softwareUsers_softwareIdIdx = "softwareUsers_software_idx";
const instances_mainSoftwareSillIdIdx = "instances_mainSoftwareSillId_idx";

export async function up(db: Kysely<any>): Promise<void> {
await db.schema.createIndex(softwares_externalIdIdx).on("softwares").column("externalId").execute();
await db.schema
.createIndex(softwares_parentExternalIdIdx)
.on("softwares")
.column("parentSoftwareWikidataId")
.execute();

await db.schema
.createIndex(software_similarExternalIdIdx)
.on("softwares__similar_software_external_datas")
.column("similarExternalId")
.execute();
await db.schema
.createIndex(software_softwareIdIdx)
.on("softwares__similar_software_external_datas")
.column("softwareId")
.execute();

await db.schema
.createIndex(compiledSoftwares_SoftwareIdIdx)
.on("compiled_softwares")
.column("softwareId")
.execute();

await db.schema
.createIndex(softwareReferents_softwareIdIdx)
.on("software_referents")
.column("softwareId")
.execute();

await db.schema.createIndex(softwareUsers_softwareIdIdx).on("software_users").column("softwareId").execute();
await db.schema.createIndex(instances_mainSoftwareSillIdIdx).on("instances").column("mainSoftwareSillId").execute();

// CREATE INDEX idx_compiled_softwares_group_by ON compiled_softwares (softwareId, annuaireCnllServiceProviders, comptoirDuLibreSoftware, latestVersion, parentWikidataSoftware, serviceProviders, similarExternalSoftwares, softwareExternalData);
await db.schema
.createIndex(compiledSoftwares_GroupByIdx)
.on("compiled_softwares")
.column("softwareId")
.column("annuaireCnllServiceProviders")
.column("comptoirDuLibreSoftware")
.column("latestVersion")
.column("serviceProviders")
.execute();
}

export async function down(db: Kysely<any>): Promise<void> {
await db.schema.dropIndex(softwares_parentExternalIdIdx).execute();
await db.schema.dropIndex(softwares_externalIdIdx).execute();
await db.schema.dropIndex(software_similarExternalIdIdx).execute();
await db.schema.dropIndex(software_softwareIdIdx).execute();
await db.schema.dropIndex(compiledSoftwares_SoftwareIdIdx).execute();
await db.schema.dropIndex(softwareReferents_softwareIdIdx).execute();
await db.schema.dropIndex(softwareUsers_softwareIdIdx).execute();
await db.schema.dropIndex(instances_mainSoftwareSillIdIdx).execute();
await db.schema.dropIndex(compiledSoftwares_GroupByIdx).execute();
}
Loading

0 comments on commit 275aaa4

Please sign in to comment.