#215 : functions to import software from wikidata

api/src/core/adapters/wikidata/getSoftwareForm.ts

@@ -0,0 +1,74 @@
+import { SoftwareFormData } from "../../usecases/readWriteSillData";
+import { createGetClaimDataValue, fetchEntity, WikidataFetchError } from "./getWikidataSoftware";
+
+export const getWikidataForm = async (wikidataId: string): Promise<SoftwareFormData | undefined> => {
+    try {
+        console.info(`   -> fetching wiki soft : ${wikidataId}`);
+        const { entity } =
+            (await fetchEntity(wikidataId).catch(error => {
+                if (error instanceof WikidataFetchError) {
+                    if (error.status === 404 || error.status === undefined) {
+                        return undefined;
+                    }
+                    throw error;
+                }
+            })) ?? {};
+
+        if (entity === undefined) {
+            return undefined;
+        }
+
+        const { getClaimDataValue } = createGetClaimDataValue({ entity });
+
+        const logoName = getClaimDataValue<"string">("P154")[0];
+
+        const license = await (async () => {
+            const licenseId = getClaimDataValue<"wikibase-entityid">("P275")[0]?.id;
+
+            if (licenseId === undefined) {
+                return undefined;
+            }
+
+            console.info(`I   -> fetching wiki license : ${licenseId}`);
+            const { entity } = await fetchEntity(licenseId).catch(() => ({ "entity": undefined }));
+
+            if (entity === undefined) {
+                return undefined;
+            }
+
+            return { "label": entity.aliases.en?.[0]?.value, "id": licenseId };
+        })();
+
+        const name =
+            entity.labels?.en?.value ?? entity.labels?.fr?.value ?? entity.labels[Object.keys(entity.labels)[0]].value;
+        const description =
+            entity.descriptions?.en?.value ??
+            entity.descriptions?.fr?.value ??
+            entity.descriptions?.[Object.keys(entity.descriptions)[0]]?.value ??
+            "";
+
+        return {
+            softwareName: name,
+            softwareDescription: description,
+            softwareType: {
+                // Todo // P306
+                type: "desktop/mobile",
+                os: { "linux": true, "windows": true, "android": false, "ios": false, "mac": false }
+            },
+            externalId: wikidataId,
+            comptoirDuLibreId: undefined,
+            softwareLicense: license?.label ?? "Copyright",
+            softwareMinimalVersion: undefined,
+            similarSoftwareExternalDataIds: [],
+            softwareLogoUrl: `https://upload.wikimedia.org/wikipedia/commons/6/69/${logoName?.replace(" ", "_") ?? ""}`,
+            softwareKeywords: [],
+            isPresentInSupportContract: false,
+            isFromFrenchPublicService: false,
+            doRespectRgaa: false
+        };
+    } catch (error) {
+        console.error(`Error for ${wikidataId} : `, error);
+        // Expected output: ReferenceError: nonExistentFunction is not defined
+        // (Note: the exact output may be browser-dependent)
+    }
+};

api/src/core/adapters/wikidata/getWikidataSoftware.ts

@@ -285,7 +285,7 @@ export class WikidataFetchError extends Error {
     }
 }
 
-async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> {
+export async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> {
     const res = await fetch(`https://www.wikidata.org/wiki/Special:EntityData/${wikidataId}.json`).catch(
         () => undefined
     );
@@ -310,7 +310,7 @@ async function fetchEntity(wikidataId: string): Promise<{ entity: Entity }> {
     return { entity };
 }
 
-function createGetClaimDataValue(params: { entity: Entity }) {
+export function createGetClaimDataValue(params: { entity: Entity }) {
     const { entity } = params;
 
     function getClaimDataValue<Type extends "string" | "wikibase-entityid" | "text-language" | "time">(

api/src/core/bootstrap.ts

@@ -20,7 +20,7 @@ import type { UserApi } from "./ports/UserApi";
 import { UseCases } from "./usecases";
 import { makeGetAgent } from "./usecases/getAgent";
 import { makeGetSoftwareFormAutoFillDataFromExternalAndOtherSources } from "./usecases/getSoftwareFormAutoFillDataFromExternalAndOtherSources";
-import { importFromHALSource } from "./usecases/importFromSource";
+import { importFromHALSource, importFromWikidataSource } from "./usecases/importFromSource";
 
 type PgDbConfig = { dbKind: "kysely"; kyselyDb: Kysely<Database> };
 
@@ -35,6 +35,7 @@ type ParamsOfBootstrapCore = {
     externalSoftwareDataOrigin: ExternalDataOrigin;
     initializeSoftwareFromSource: boolean;
     botAgentEmail: string | undefined;
+    listToImport?: string[];
 };
 
 export type Context = {
@@ -68,7 +69,8 @@ export async function bootstrapCore(
         doPerformCacheInitialization,
         externalSoftwareDataOrigin,
         initializeSoftwareFromSource,
-        botAgentEmail
+        botAgentEmail,
+        listToImport
     } = params;
 
     const { getSoftwareLatestVersion } = createGetSoftwareLatestVersion({
@@ -131,6 +133,17 @@ export async function bootstrapCore(
 
             console.log(" ------ Feeding database with HAL software finished ------");
         }
+        if (externalSoftwareDataOrigin === "wikidata") {
+            console.log(" ------ Feeding database with Wikidata software started ------");
+            if (!botAgentEmail) throw new Error("No bot agent email provided");
+            const importWikidata = importFromWikidataSource(dbApi);
+            try {
+                await importWikidata(botAgentEmail, listToImport ?? []);
+            } catch (err) {
+                console.error(err);
+            }
+            console.log(" ------ Feeding database with Wikidata software finished ------");
+        }
     }
 
     if (doPerPerformPeriodicalCompilation) {

api/src/core/usecases/importFromSource.ts

@@ -1,6 +1,7 @@
 import { DbApiV2 } from "../ports/DbApiV2";
 import { halAPIGateway } from "../adapters/hal/HalAPI";
 import { halRawSoftwareToSoftwareForm } from "../adapters/hal/getSoftwareForm";
+import { getWikidataForm } from "../adapters/wikidata/getSoftwareForm";
 
 export const importFromHALSource: (dbApi: DbApiV2) => (agentEmail: string) => Promise<Promise<number | undefined>[]> = (
     dbApi: DbApiV2
@@ -35,3 +36,40 @@ export const importFromHALSource: (dbApi: DbApiV2) => (agentEmail: string) => Pr
         });
     };
 };
+
+export const importFromWikidataSource: (
+    dbApi: DbApiV2
+) => (agentEmail: string, softwareIds: string[]) => Promise<Promise<number | undefined>[]> = (dbApi: DbApiV2) => {
+    return async (agentEmail: string, softwareIds: string[]) => {
+        const agent = await dbApi.agent.getByEmail(agentEmail);
+        const agentId = agent
+            ? agent.id
+            : await dbApi.agent.add({
+                  email: agentEmail,
+                  "isPublic": false,
+                  organization: "",
+                  about: "This is an bot user created to import data."
+              });
+
+        const dbSoftwares = await dbApi.software.getAll();
+        const dbSoftwaresNames = dbSoftwares.map(software => {
+            return software.softwareName;
+        });
+
+        return softwareIds.map(async (softwareId: string) => {
+            const newSoft = await getWikidataForm(softwareId);
+            if (!newSoft) {
+                return -1;
+            }
+
+            const index = dbSoftwaresNames.indexOf(newSoft?.softwareName ?? "");
+
+            if (index != -1) {
+                return dbSoftwares[index].softwareId;
+            } else {
+                console.log("Importing wikidata : ", softwareId);
+                return dbApi.software.create({ formData: newSoft, externalDataOrigin: "wikidata", agentId: agentId });
+            }
+        });
+    };
+};

api/src/env.ts

@@ -30,7 +30,8 @@ const zConfiguration = z.object({
     "externalSoftwareDataOrigin": z.enum(["wikidata", "HAL"]).optional(),
     "databaseUrl": z.string(),
     "initializeSoftwareFromSource": z.boolean(),
-    "botAgentEmail": z.string().optional()
+    "botAgentEmail": z.string().optional(),
+    "listToImport": z.array(z.string()).optional()
 });
 
 const getJsonConfiguration = () => {
@@ -69,7 +70,10 @@ const getJsonConfiguration = () => {
         "redirectUrl": process.env.SILL_REDIRECT_URL,
         "databaseUrl": process.env.DATABASE_URL,
         "initializeSoftwareFromSource": process.env.INIT_SOFT_FROM_SOURCE?.toLowerCase() === "true",
-        "botAgentEmail": process.env?.BOT_AGENT_EMAIL
+        "botAgentEmail": process.env?.BOT_AGENT_EMAIL,
+        "listToImport": process.env?.SILL_IMPORT_WIKIDATA
+            ? JSON.parse(process.env?.SILL_IMPORT_WIKIDATA)?.ids
+            : undefined
     };
 };
 

api/src/rpc/start.ts

@@ -43,6 +43,7 @@ export async function startRpcService(params: {
     databaseUrl: string;
     initializeSoftwareFromSource: boolean;
     botAgentEmail?: string;
+    listToImport?: string[];
 }) {
     const {
         redirectUrl,
@@ -57,6 +58,7 @@ export async function startRpcService(params: {
         databaseUrl,
         botAgentEmail,
         initializeSoftwareFromSource,
+        listToImport,
         ...rest
     } = params;
 
@@ -90,7 +92,8 @@ export async function startRpcService(params: {
         "doPerformCacheInitialization": redirectUrl === undefined,
         "externalSoftwareDataOrigin": externalSoftwareDataOrigin,
         "botAgentEmail": botAgentEmail,
-        "initializeSoftwareFromSource": initializeSoftwareFromSource
+        "initializeSoftwareFromSource": initializeSoftwareFromSource,
+        "listToImport": listToImport ?? []
     });
 
     console.log("Core API initialized");

deployments/docker-compose-example/.env.sample

@@ -12,13 +12,14 @@ SILL_GITHUB_TOKEN=xxxxx
 SILL_API_PORT=3084
 SILL_IS_DEV_ENVIRONNEMENT=true
 SILL_EXTERNAL_SOFTWARE_DATA_ORIGIN=wikidata
+SILL_IMPORT_WIKIDATA=|
+        { 
+          ids: ['Q10135']
+        }
 INIT_SOFT_FROM_SOURCE=false
 [email protected]
 
 DATABASE_URL=postgresql://sill:pg_password@localhost:5432/sill
-POSTGRES_DB=sill
-POSTGRES_USER=sill
-POSTGRES_PASSWORD=pg_password
 
 # this is only for the script load-git-repo-in-pg.ts, not needed for api any more
 [email protected]:codegouvfr/sill-data-test.git