feat: Implement workers for parallel imports and settings (#16) (#17)

deps.ts

@@ -4,6 +4,7 @@ export * as Dom from "https://deno.land/x/[email protected]/deno-dom-wasm.ts
 export * as Oak from "https://deno.land/x/[email protected]/mod.ts";
 export { default as parseIngredient } from "https://cdn.skypack.dev/[email protected]?min";
 export * from "https://cdn.skypack.dev/[email protected]?min";
+export * as Zod from "https://cdn.skypack.dev/[email protected]?dts";
 
 import {
   format,

lock.json

@@ -28,13 +28,24 @@
   "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=imports,min/optimized/numeric-quantity.js": "9bc978ab75638abc08952c28e2a197ba6a9650330765b2c3b2bd4903edb296f5",
   "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=imports,min/optimized/parse-ingredient.js": "76f523604ed6a089b871e8bff12212a4743256eb1a25e669b8decd16e3fb8f54",
   "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=raw,min/schema.d.ts": "df5ba0a1c963345628b241d51db80b3dc08677a098e5cc40ba751ec64604fe8b",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=imports/optimized/zod.js": "e2b823949d9857fa57559bddea64383b41b24b121d73aea1b4a3f85f5fde359f",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/PseudoPromise.d.ts": "cfa818a0d73d80b2d92dbb61d99030a45b5bc7d71ebc0532bf2c0c9625ae6849",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/ZodError.d.ts": "cee80c93a48a85f27db58058cddc746ffa7e12d66ff7f429ee446dd1bd37b854",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/external.d.ts": "c8533da6c8d66a46ba851740f7c67605b0c0a15e59957854fe39e09b2e2d305a",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/helpers/errorUtil.d.ts": "8196d19e37f8d5eaa46bd4c040fd18a1653d1823a0b5c0aad71977a1d4cb340f",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/helpers/parseUtil.d.ts": "22897c908ca3d84d79a987900e3e780e98d501ee742d236ee10e870e8bae6d2f",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/helpers/partialUtil.d.ts": "dd976cb6402b7964cecea53f8cb4786ae41872079ca8bcbefbaa6eff6d7380a9",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/helpers/util.d.ts": "a2da1b91bdc9b8c508e664b4b43a572ba6a563f0d7eb77ebb416b23fd0e7d2f4",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/index.d.ts": "b9f8d8aa658f32993c81d2ccf6afd1848e6db09535ffc6b743ac3d1a7590153d",
+  "https://cdn.skypack.dev/-/[email protected]/dist=es2020,mode=types/lib/types.d.ts": "13ada06cf36b95ed99668acd6339343553d480ec26e4690e63246fd3ca8dd544",
   "https://cdn.skypack.dev/[email protected]/js/src/collapse.js": "086bf931f34cbf4f4fa5782bffdd9daf5ea0ef909b4d2eb10a133e51b43278d8",
   "https://cdn.skypack.dev/[email protected]/js/src/dropdown.js": "86d5250b16e701348f30d7a36dc07a687f87894b03bcce993c142bb7019c3acc",
   "https://cdn.skypack.dev/[email protected]/locale/de?min": "d3b1f183ed2b3f71d6dc2c45fd29c6dee33d6137beea7e12c938047958c46c5c",
   "https://cdn.skypack.dev/[email protected]/locale/en-US?min": "8ff247ce9a8a4c4f6ec2194e72dfa1ac8370297579550da5028ee0b758363dda",
   "https://cdn.skypack.dev/[email protected]?min": "07bf56907e6c0e8e2e9abb4956088dae22261497962be496da9525cee283a2a2",
   "https://cdn.skypack.dev/[email protected]?min": "50e7b35d7a13cc52a3eaccbed3ca29b8ab881218cb2c8921fe1f60f836a5864d",
   "https://cdn.skypack.dev/[email protected]?min": "223e1dee9b0f7d71eaf2fb0dc86fe389b2bd46b25c725d7fb37ec2edd3ae08fb",
+  "https://cdn.skypack.dev/[email protected]?dts": "37379a5ddbf56b155947760d7cb70f4d9597bb60d2e999f83eedee9265e4afdb",
   "https://deno.land/[email protected]/_util/assert.ts": "e1f76e77c5ccb5a8e0dbbbe6cce3a56d2556c8cb5a9a8802fc9565af72462149",
   "https://deno.land/[email protected]/fs/_util.ts": "68508c05d5a02678179a02beabf2b3eac5b16c5a9cbd1c272686d8101bb2679d",
   "https://deno.land/[email protected]/fs/copy.ts": "b562e8f482cb8459bb011cbd769769bbdb4f6bc966effd277c06edbdbe41b72e",

src/data/parse/import/import_recipe.ts

@@ -0,0 +1,51 @@
+import { Recipe } from "../../model/recipe.ts";
+import { ImportRecipeRequest, ImportRecipeResponse } from "./types.ts";
+
+export interface ImportResult {
+  url: string;
+  success: boolean;
+  error?: string;
+  recipe?: Recipe;
+}
+
+export function importRecipes(
+  args: {
+    urls: string[];
+    configDir: string;
+    importWorkerCount: number | null;
+  },
+): Promise<ImportResult[]> {
+  const results: ImportResult[] = [];
+  let running = 0;
+
+  return new Promise((resolve) => {
+    const workerDone = (e: MessageEvent<ImportRecipeResponse>) => {
+      running--;
+      results.push(e.data);
+      if (running === 0) {
+        resolve(results);
+      }
+    };
+
+    // TODO control how many workers are spawned
+    for (const url of args.urls) {
+      const worker = new Worker(
+        new URL("./import_worker.ts", import.meta.url).href,
+        {
+          type: "module",
+          // @ts-ignore IntelliJ hiccup
+          deno: {
+            namespace: true,
+          },
+        },
+      );
+      worker.onmessage = workerDone;
+      const request: ImportRecipeRequest = {
+        url: url.trim(),
+        configDir: args.configDir,
+      };
+      worker.postMessage(request);
+      running++;
+    }
+  });
+}

src/data/parse/import_recipe.ts → src/data/parse/import/import_worker.ts

@@ -2,37 +2,27 @@ import {
   SchemaAggregateRating,
   SchemaNutritionInformation,
   SchemaReview,
-} from "../../../deps.ts";
-import { Recipe, Review } from "../model/recipe.ts";
-import { Tag } from "../model/tag.ts";
-import { downloadThumbnail, fetchCustom } from "../util/thumbnails.ts";
-import { durationToSeconds, parseDuration } from "./duration.ts";
-import { SchemaParser } from "./schema_parser.ts";
-import { ensureArray, extractNumber, first } from "./util.ts";
+} from "../../../../deps.ts";
+import { Recipe, Review } from "../../model/recipe.ts";
+import { Tag } from "../../model/tag.ts";
+import { downloadThumbnail, fetchCustom } from "../../util/thumbnails.ts";
+import { durationToSeconds, parseDuration } from "../duration.ts";
+import { SchemaParser } from "../schema_parser.ts";
+import { ensureArray, extractNumber, first } from "../util.ts";
+import { ImportRecipeRequest, ImportRecipeResponse } from "./types.ts";
 
-export interface ImportResult {
-  url: string;
-  success: boolean;
-  error?: string;
-  recipe?: Recipe;
-}
-
-export async function importRecipes(
-  urls: string[],
-  configDir: string,
-): Promise<ImportResult[]> {
-  const results: ImportResult[] = [];
-  for (const url of urls) {
-    const imported = await importRecipe(url.trim(), configDir);
-    results.push({
-      url,
-      success: typeof imported !== "string",
-      recipe: typeof imported === "string" ? undefined : imported,
-      error: typeof imported === "string" ? imported : undefined,
-    });
-  }
-  return results;
-}
+self.onmessage = function (e: MessageEvent<ImportRecipeRequest>) {
+  importRecipe(e.data.url, e.data.configDir).then((result) => {
+    const response: ImportRecipeResponse = {
+      url: e.data.url,
+      success: typeof result !== "string",
+      recipe: typeof result === "string" ? undefined : result,
+      error: typeof result === "string" ? result : undefined,
+    };
+    self.postMessage(response);
+    self.close();
+  });
+};
 
 export async function importRecipe(
   url: string,

src/data/parse/import/types.ts

@@ -0,0 +1,13 @@
+import { Recipe } from "../../model/recipe.ts";
+
+export interface ImportRecipeRequest {
+  url: string;
+  configDir: string;
+}
+
+export interface ImportRecipeResponse {
+  url: string;
+  success: boolean;
+  recipe?: Recipe;
+  error?: string;
+}

src/http/routes/recipe.routes.ts

@@ -1,6 +1,6 @@
 import { fs, Oak, path } from "../../../deps.ts";
 import { Recipe } from "../../data/model/recipe.ts";
-import { importRecipes } from "../../data/parse/import_recipe.ts";
+import { importRecipes } from "../../data/parse/import/import_recipe.ts";
 import { RecipeService } from "../../data/service/recipe.service.ts";
 import { toNumber } from "../../data/util/convert.ts";
 import {
@@ -139,10 +139,11 @@ router
       if (!urls) {
         return await next();
       }
-      const results = await importRecipes(
-        urls!.split("\n"),
-        ctx.configDir(),
-      );
+      const results = await importRecipes({
+        urls: urls!.split("\n"),
+        configDir: ctx.state.configDir,
+        importWorkerCount: ctx.state.settings.importWorkerCount,
+      });
       const service = ctx.state.services.RecipeService;
       service.create(results.filter((r) => r.success).map((r) => r.recipe!));
       await ctx.render(RecipeImportTemplate, { results });
@@ -183,7 +184,7 @@ router
         const formDataReader: Oak.FormDataReader = await ctx.request.body({
           type: "form-data",
         }).value;
-        await assignRecipeFields(formDataReader, recipe, ctx.configDir());
+        await assignRecipeFields(formDataReader, recipe, ctx.state.configDir);
         service.update([recipe]);
         ctx.response.redirect(
           urlWithParams(UrlHelper.INSTANCE.recipe(recipe), {
@@ -207,7 +208,7 @@ router
       const formDataReader: Oak.FormDataReader = await ctx.request.body({
         type: "form-data",
       }).value;
-      await assignRecipeFields(formDataReader, recipe, ctx.configDir());
+      await assignRecipeFields(formDataReader, recipe, ctx.state.configDir);
       service.create([recipe]);
       ctx.response.redirect(
         urlWithParams(UrlHelper.INSTANCE.recipe(recipe), {
@@ -242,7 +243,7 @@ router
       if (!recipe) {
         await next();
       } else {
-        await deleteThumbnail(recipe, ctx.configDir());
+        await deleteThumbnail(recipe, ctx.state.configDir);
         service.delete([recipe]);
         ctx.response.redirect(
           urlWithParams(UrlHelper.INSTANCE.recipeList(), {

src/http/routes/thumbnails.routes.ts

@@ -1,15 +1,19 @@
-import { getThumbnailDir } from "../../data/util/thumbnails.ts";
 import { Oak } from "../../../deps.ts";
+import { getThumbnailDir } from "../../data/util/thumbnails.ts";
+import { AppState } from "../webserver.ts";
 
 const router: Oak.Router = new Oak.Router();
-router.get("/thumbnails/(.+)", async (ctx, next) => {
-  try {
-    await Oak.send(ctx, ctx.params[0]!, {
-      root: getThumbnailDir(ctx.configDir()),
-    });
-  } catch {
-    await next();
-  }
-});
+router.get(
+  "/thumbnails/(.+)",
+  async (ctx: Oak.RouterContext<{ 0: string }, AppState>, next) => {
+    try {
+      await Oak.send(ctx, ctx.params[0]!, {
+        root: getThumbnailDir(ctx.state.configDir),
+      });
+    } catch {
+      await next();
+    }
+  },
+);
 
 export { router as ThumbnailsRouter };

src/http/webserver.ts

@@ -1,7 +1,7 @@
 import { log, Oak } from "../../deps.ts";
 import { Database } from "../data/db.ts";
 import { Services, servicesFactory } from "../data/service/services.ts";
-import { configDirAdapter } from "./adapters/config_dir_adapter.ts";
+import { Settings } from "../settings.ts";
 import { orderByAdapter } from "./adapters/order_by_adapter.ts";
 import { paginationAdapter } from "./adapters/pagination_adapter.ts";
 import { parameterAdapter } from "./adapters/parameter_adapter.ts";
@@ -11,11 +11,17 @@ import { Routers } from "./routes/routers.ts";
 
 export interface AppState {
   services: Services;
+  settings: Settings;
+  configDir: string;
 }
 
-function buildState(db: Database): AppState {
+function buildState(
+  args: { db: Database; settings: Settings; configDir: string },
+): AppState {
   return {
-    services: servicesFactory(db),
+    services: servicesFactory(args.db),
+    settings: args.settings,
+    configDir: args.configDir,
   };
 }
 
@@ -29,13 +35,17 @@ export async function spawnServer(
     cert?: string;
     configDir: string;
     db: Database;
+    settings: Settings;
   },
 ) {
-  const state: AppState = buildState(args.db);
+  const state: AppState = buildState({
+    db: args.db,
+    settings: args.settings,
+    configDir: args.configDir,
+  });
 
   const app = new Oak.Application<AppState>({ state });
   app.use(
-    configDirAdapter(args.configDir),
     parameterAdapter(),
     orderByAdapter(),
     templateAdapter(args.debug),

src/main.ts

@@ -1,6 +1,7 @@
-import { Database } from "./data/db.ts";
 import { Cliffy, Colors, fs, log, LogRecord, path } from "../deps.ts";
+import { Database } from "./data/db.ts";
 import { spawnServer } from "./http/webserver.ts";
+import { readFromDisk, Settings } from "./settings.ts";
 import { DEFAULT_CONFIG_DIR, defaultConfigDir } from "./util.ts";
 
 interface Options {
@@ -76,17 +77,27 @@ async function setupLogger(debug?: boolean) {
   });
 }
 
-async function main(): Promise<void> {
+async function main(): Promise<number> {
   const options = await parseOptions();
   await prepareConfigDir(options);
   await setupLogger(options.debug);
+  let settings: Settings;
+  try {
+    settings = await readFromDisk(options.configDir);
+  } catch (e) {
+    log.error(e);
+    return 1;
+  }
   const database = new Database(options.configDir);
   database.migrate();
 
   await spawnServer({
     ...options,
     db: database,
+    settings,
   });
+
+  return 0;
 }
 
 if (import.meta.main) {

src/settings.ts

@@ -0,0 +1,50 @@
+import { fs, log, path, Zod as z } from "../deps.ts";
+import { getCpuCores } from "./util.ts";
+
+export interface Settings {
+  /**
+   * The number of workers to spawn concurrently when importing Recipes.
+   * @default number of CPU cores on the system
+   */
+  importWorkerCount: number | null;
+
+  // TODO doc & impl
+  ingredientSortMode?: unknown;
+}
+
+export const DEFAULT_SETTINGS: Settings = {
+  importWorkerCount: null,
+};
+const CPU_CORES = getCpuCores();
+const Schema = z.object({
+  importWorkerCount: z.number().refine(
+    (val?: number) =>
+      CPU_CORES === undefined || val === undefined ||
+      val > 0 && val <= CPU_CORES,
+    (val?: number) => ({
+      message:
+        `importWorkerCount: Value ${val} must be greater 0 and lower than number of CPU cores available, i.e. ${getCpuCores()}.`,
+    }),
+  ),
+  ingredientSortMode: z.number().optional(),
+});
+
+export const SETTINGS_FILENAME = "settings.json";
+
+export async function readFromDisk(configDir: string): Promise<Settings> {
+  const file = path.join(configDir, SETTINGS_FILENAME);
+  if (await fs.exists(file)) {
+    const contents = await Deno.readTextFile(file);
+    try {
+      return Schema.parse(JSON.parse(contents));
+    } catch (e) {
+      throw new Error(`Error reading ${SETTINGS_FILENAME}: ${e}`);
+    }
+  }
+  log.debug(() =>
+    `Could not find settings file ${SETTINGS_FILENAME}. Using default settings: ${
+      JSON.stringify(DEFAULT_SETTINGS)
+    }`
+  );
+  return DEFAULT_SETTINGS;
+}