feat: Add more settings (#17)

src/data/parse/import/import_recipe.ts

@@ -1,4 +1,3 @@
-import { getCpuCores } from "../../../util.ts";
 import { Recipe } from "../../model/recipe.ts";
 import { ImportRecipeRequest, ImportRecipeResponse } from "./types.ts";
 
@@ -13,7 +12,8 @@ export function importRecipes(
   args: {
     urls: string[];
     configDir: string;
-    importWorkerCount: number | null;
+    importWorkerCount: number;
+    userAgent: string;
   },
 ): Promise<ImportResult[]> {
   const results: ImportResult[] = [];
@@ -26,10 +26,9 @@ export function importRecipes(
 
     const workers: Worker[] = [];
     const jobs: string[] = [...args.urls];
-    const cores = getCpuCores();
     const workerCount = Math.min(
       args.urls.length,
-      args.importWorkerCount || cores || 1,
+      args.importWorkerCount,
     );
 
     const workerDone = (
@@ -56,6 +55,7 @@ export function importRecipes(
       const request: ImportRecipeRequest = {
         url: url.trim(),
         configDir: args.configDir,
+        userAgent: args.userAgent,
       };
       worker.postMessage(request);
       pending++;

src/data/parse/import/import_worker.ts

@@ -12,24 +12,27 @@ import { ensureArray, extractNumber, first } from "../util.ts";
 import { ImportRecipeRequest, ImportRecipeResponse } from "./types.ts";
 
 self.onmessage = function (e: MessageEvent<ImportRecipeRequest>) {
-  importRecipe(e.data.url, e.data.configDir).then((result) => {
-    const response: ImportRecipeResponse = {
-      url: e.data.url,
-      success: typeof result !== "string",
-      recipe: typeof result === "string" ? undefined : result,
-      error: typeof result === "string" ? result : undefined,
-    };
-    self.postMessage(response);
-  });
+  importRecipe(e.data.url, e.data.configDir, e.data.userAgent).then(
+    (result) => {
+      const response: ImportRecipeResponse = {
+        url: e.data.url,
+        success: typeof result !== "string",
+        recipe: typeof result === "string" ? undefined : result,
+        error: typeof result === "string" ? result : undefined,
+      };
+      self.postMessage(response);
+    },
+  );
 };
 
 export async function importRecipe(
   url: string,
   configDir: string,
+  userAgent: string,
 ): Promise<Recipe | string> {
   let html: string;
   try {
-    const response = await fetchCustom(url);
+    const response = await fetchCustom(url, userAgent);
     html = new TextDecoder().decode(
       new Uint8Array(await response.arrayBuffer()),
     );
@@ -72,6 +75,7 @@ export async function importRecipe(
     source: url,
     thumbnail: await downloadThumbnail(
       configDir,
+      userAgent,
       first(schemaRecipe.image as string),
     ),
     prepTime: schemaRecipe.prepTime

src/data/parse/import/types.ts

@@ -3,6 +3,7 @@ import { Recipe } from "../../model/recipe.ts";
 export interface ImportRecipeRequest {
   url: string;
   configDir: string;
+  userAgent: string;
 }
 
 export interface ImportRecipeResponse {

src/data/util/thumbnails.ts

@@ -17,6 +17,7 @@ export function getUniqueFilename(dir: string, origFilename: string): string {
 
 export async function downloadThumbnail(
   configDir: string,
+  userAgent: string,
   url?: string,
 ): Promise<string | undefined> {
   if (!url) {
@@ -28,7 +29,7 @@ export async function downloadThumbnail(
     path.basename(new URL(url).pathname),
   );
   log.debug(() => `[Thumbnail] Downloading ${url} as ${filename}`);
-  const response = await fetchCustom(url);
+  const response = await fetchCustom(url, userAgent);
   await Deno.writeFile(
     path.join(thumbnailDir, filename),
     new Uint8Array(await response.arrayBuffer()),
@@ -53,12 +54,11 @@ const merge = (target: any, source: any) => {
 };
 
 /**
- * A custom wrapper around fetch to make sure requests use the Googlebot User Agent.
- * @param input
- * @param init
+ * A custom wrapper around fetch to make sure requests use the configured User Agent.
  */
 export function fetchCustom(
   input: RequestInfo,
+  userAgent: string,
   init?: RequestInit,
 ): Promise<Response> {
   return fetch(
@@ -67,10 +67,7 @@ export function fetchCustom(
       init ?? {},
       {
         headers: {
-          // TODO make configurable
-          "User-Agent":
-            // "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0",
+          "User-Agent": userAgent,
         },
       },
     ),

src/http/routes/recipe.routes.ts

@@ -145,6 +145,7 @@ router
         urls: urls!.split("\n"),
         configDir: ctx.state.configDir,
         importWorkerCount: ctx.state.settings.importWorkerCount,
+        userAgent: ctx.state.settings.userAgent,
       });
       const service = services.get(RecipeService);
       service.create(results.filter((r) => r.success).map((r) => r.recipe!));

src/settings.ts

@@ -1,20 +1,65 @@
 import { fs, log, path, Zod as z } from "../deps.ts";
 import { getCpuCores } from "./util.ts";
 
+export const DEFAULT_USER_AGENT =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0";
+
+/**
+ * Specifies in which order ingredients are sorted on the recipe detail page.
+ * "FULL" is used to denote ingredients where both a unit and a measurement is present (e.g. "100g")
+ * "UNITLESS" is used to denote ingredients without a unit (e.g. "3")
+ * "EMPTY" is used to denote ingredients with neither a unit nor a measurement (e.g. "Water")
+ */
+export enum IngredientSortOrder {
+  /**
+   * List them in the order they are stored.
+   */
+  ORIGINAL = "ORIGINAL",
+
+  FULL_UNITLESS_EMPTY = "FULL_UNITLESS_EMPTY",
+  UNITLESS_EMPTY_FULL = "UNITLESS_EMPTY_FULL",
+  EMPTY_UNITLESS_FULL = "EMPTY_UNITLESS_FULL",
+  UNITLESS_FULL_EMPTY = "UNITLESS_FULL_EMPTY",
+  EMPTY_FULL_UNITLESS = "EMPTY_FULL_UNITLESS",
+  FULL_EMPTY_UNITLESS = "FULL_EMPTY_UNITLESS",
+}
+
+export const DEFAULT_INGREDIENT_POSTPROCESSING = [
+  "TL",
+  "EL",
+  "dl",
+  "kl.",
+  "gr.",
+];
+
 export interface Settings {
   /**
    * The number of workers to spawn concurrently when importing Recipes.
    * @default number of CPU cores on the system
    */
-  importWorkerCount: number | null;
+  importWorkerCount: number;
+
+  /**
+   * Configure in which order ingredients are sorted.
+   * @see IngredientSortOrder#FULL_UNITLESS_EMPTY
+   */
+  ingredientSortOrder: IngredientSortOrder;
+
+  /**
+   * The ingredient extraction parser does not work one hundred percent for all locales.
+   * Strings may be specified here which will be moved from an ingredient's description to its unit.
+   * @see DEFAULT_INGREDIENT_POSTPROCESSING
+   */
+  ingredientUnitPostprocessing: string[];
 
-  // TODO doc & impl
-  ingredientSortMode?: unknown;
+  /**
+   * The User Agent to use when doing HTTP Requests.
+   * @see #DEFAULT_USER_AGENT
+   */
+  userAgent: string;
 }
 
-export const DEFAULT_SETTINGS: Settings = {
-  importWorkerCount: null,
-};
+export const DEFAULT_SETTINGS: Partial<Settings> = {};
 const CPU_CORES = getCpuCores();
 const Schema = z.object({
   importWorkerCount: z.number().refine(
@@ -25,8 +70,20 @@ const Schema = z.object({
       message:
         `importWorkerCount: Value ${val} must be greater 0 and lower than number of CPU cores available, i.e. ${getCpuCores()}.`,
     }),
+  ).optional().default(CPU_CORES || 1),
+  ingredientSortOrder: z.string().refine(
+    (val: string) =>
+      Boolean(IngredientSortOrder[val as unknown as IngredientSortOrder]),
+    (val: string) => ({
+      message: `ingredientSortOrder: Value ${val} must be one of ${
+        Object.values(IngredientSortOrder)
+      }`,
+    }),
+  ).optional().default(IngredientSortOrder.FULL_UNITLESS_EMPTY),
+  ingredientUnitPostprocessing: z.array(z.string()).optional().default(
+    DEFAULT_INGREDIENT_POSTPROCESSING,
   ),
-  ingredientSortMode: z.number().optional(),
+  userAgent: z.string().optional().default(DEFAULT_USER_AGENT),
 });
 
 export const SETTINGS_FILENAME = "settings.json";
@@ -36,7 +93,7 @@ export async function readFromDisk(configDir: string): Promise<Settings> {
   if (await fs.exists(file)) {
     const contents = await Deno.readTextFile(file);
     try {
-      return Schema.parse(JSON.parse(contents));
+      return Schema.parse(JSON.parse(contents)) as Settings;
     } catch (e) {
       throw new Error(`Error reading ${SETTINGS_FILENAME}: ${e}`);
     }
@@ -46,5 +103,5 @@ export async function readFromDisk(configDir: string): Promise<Settings> {
       JSON.stringify(DEFAULT_SETTINGS)
     }`
   );
-  return DEFAULT_SETTINGS;
+  return Schema.parse(DEFAULT_SETTINGS) as Settings;
 }

src/tpl/helpers/helpers.ts

@@ -1,15 +1,23 @@
 import { UrlHelper } from "../../http/url_helper.ts";
+import { AppState } from "../../http/webserver.ts";
 import { AssetsHelper } from "./assets_helper.ts";
 import { FormatHelper } from "./format_helper.ts";
 import { IngredientHelper } from "./ingredient_helper.ts";
 import { TranslationHelper } from "./translation_helper.ts";
 
-export const Helpers = {
-  ...AssetsHelper.INSTANCE.api,
-  ...TranslationHelper.INSTANCE.api,
-  ...IngredientHelper.INSTANCE.api,
-  ...FormatHelper.INSTANCE.api,
-  ...{
-    u: UrlHelper.INSTANCE,
-  },
-};
+export type Helpers = AssetsHelper | TranslationHelper | IngredientHelper | FormatHelper | { u: UrlHelper };
+
+export function helperFactory(appState: AppState): Helpers {
+  return {
+    ...AssetsHelper.INSTANCE.api,
+    ...TranslationHelper.INSTANCE.api,
+    ...new IngredientHelper(
+      appState.settings.ingredientSortOrder,
+      appState.settings.ingredientUnitPostprocessing,
+    ).api,
+    ...FormatHelper.INSTANCE.api,
+    ...{
+      u: UrlHelper.INSTANCE,
+    },
+  };
+}