diff --git a/src/data/parse/import/import_recipe.ts b/src/data/parse/import/import_recipe.ts index 5b4d984..3044f7b 100644 --- a/src/data/parse/import/import_recipe.ts +++ b/src/data/parse/import/import_recipe.ts @@ -1,4 +1,3 @@ -import { getCpuCores } from "../../../util.ts"; import { Recipe } from "../../model/recipe.ts"; import { ImportRecipeRequest, ImportRecipeResponse } from "./types.ts"; @@ -13,7 +12,8 @@ export function importRecipes( args: { urls: string[]; configDir: string; - importWorkerCount: number | null; + importWorkerCount: number; + userAgent: string; }, ): Promise { const results: ImportResult[] = []; @@ -26,10 +26,9 @@ export function importRecipes( const workers: Worker[] = []; const jobs: string[] = [...args.urls]; - const cores = getCpuCores(); const workerCount = Math.min( args.urls.length, - args.importWorkerCount || cores || 1, + args.importWorkerCount, ); const workerDone = ( @@ -56,6 +55,7 @@ export function importRecipes( const request: ImportRecipeRequest = { url: url.trim(), configDir: args.configDir, + userAgent: args.userAgent, }; worker.postMessage(request); pending++; diff --git a/src/data/parse/import/import_worker.ts b/src/data/parse/import/import_worker.ts index 1d914ef..6ce1547 100644 --- a/src/data/parse/import/import_worker.ts +++ b/src/data/parse/import/import_worker.ts @@ -12,24 +12,27 @@ import { ensureArray, extractNumber, first } from "../util.ts"; import { ImportRecipeRequest, ImportRecipeResponse } from "./types.ts"; self.onmessage = function (e: MessageEvent) { - importRecipe(e.data.url, e.data.configDir).then((result) => { - const response: ImportRecipeResponse = { - url: e.data.url, - success: typeof result !== "string", - recipe: typeof result === "string" ? undefined : result, - error: typeof result === "string" ? result : undefined, - }; - self.postMessage(response); - }); + importRecipe(e.data.url, e.data.configDir, e.data.userAgent).then( + (result) => { + const response: ImportRecipeResponse = { + url: e.data.url, + success: typeof result !== "string", + recipe: typeof result === "string" ? undefined : result, + error: typeof result === "string" ? result : undefined, + }; + self.postMessage(response); + }, + ); }; export async function importRecipe( url: string, configDir: string, + userAgent: string, ): Promise { let html: string; try { - const response = await fetchCustom(url); + const response = await fetchCustom(url, userAgent); html = new TextDecoder().decode( new Uint8Array(await response.arrayBuffer()), ); @@ -72,6 +75,7 @@ export async function importRecipe( source: url, thumbnail: await downloadThumbnail( configDir, + userAgent, first(schemaRecipe.image as string), ), prepTime: schemaRecipe.prepTime diff --git a/src/data/parse/import/types.ts b/src/data/parse/import/types.ts index feeaaea..66978b6 100644 --- a/src/data/parse/import/types.ts +++ b/src/data/parse/import/types.ts @@ -3,6 +3,7 @@ import { Recipe } from "../../model/recipe.ts"; export interface ImportRecipeRequest { url: string; configDir: string; + userAgent: string; } export interface ImportRecipeResponse { diff --git a/src/data/util/thumbnails.ts b/src/data/util/thumbnails.ts index 25bb209..1cd90d9 100644 --- a/src/data/util/thumbnails.ts +++ b/src/data/util/thumbnails.ts @@ -17,6 +17,7 @@ export function getUniqueFilename(dir: string, origFilename: string): string { export async function downloadThumbnail( configDir: string, + userAgent: string, url?: string, ): Promise { if (!url) { @@ -28,7 +29,7 @@ export async function downloadThumbnail( path.basename(new URL(url).pathname), ); log.debug(() => `[Thumbnail] Downloading ${url} as ${filename}`); - const response = await fetchCustom(url); + const response = await fetchCustom(url, userAgent); await Deno.writeFile( path.join(thumbnailDir, filename), new Uint8Array(await response.arrayBuffer()), @@ -53,12 +54,11 @@ const merge = (target: any, source: any) => { }; /** - * A custom wrapper around fetch to make sure requests use the Googlebot User Agent. - * @param input - * @param init + * A custom wrapper around fetch to make sure requests use the configured User Agent. */ export function fetchCustom( input: RequestInfo, + userAgent: string, init?: RequestInit, ): Promise { return fetch( @@ -67,10 +67,7 @@ export function fetchCustom( init ?? {}, { headers: { - // TODO make configurable - "User-Agent": - // "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0", + "User-Agent": userAgent, }, }, ), diff --git a/src/http/routes/recipe.routes.ts b/src/http/routes/recipe.routes.ts index a9ffc81..cd7e876 100644 --- a/src/http/routes/recipe.routes.ts +++ b/src/http/routes/recipe.routes.ts @@ -145,6 +145,7 @@ router urls: urls!.split("\n"), configDir: ctx.state.configDir, importWorkerCount: ctx.state.settings.importWorkerCount, + userAgent: ctx.state.settings.userAgent, }); const service = services.get(RecipeService); service.create(results.filter((r) => r.success).map((r) => r.recipe!)); diff --git a/src/settings.ts b/src/settings.ts index b231be3..3f9f8ff 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1,20 +1,65 @@ import { fs, log, path, Zod as z } from "../deps.ts"; import { getCpuCores } from "./util.ts"; +export const DEFAULT_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0"; + +/** + * Specifies in which order ingredients are sorted on the recipe detail page. + * "FULL" is used to denote ingredients where both a unit and a measurement is present (e.g. "100g") + * "UNITLESS" is used to denote ingredients without a unit (e.g. "3") + * "EMPTY" is used to denote ingredients with neither a unit nor a measurement (e.g. "Water") + */ +export enum IngredientSortOrder { + /** + * List them in the order they are stored. + */ + ORIGINAL = "ORIGINAL", + + FULL_UNITLESS_EMPTY = "FULL_UNITLESS_EMPTY", + UNITLESS_EMPTY_FULL = "UNITLESS_EMPTY_FULL", + EMPTY_UNITLESS_FULL = "EMPTY_UNITLESS_FULL", + UNITLESS_FULL_EMPTY = "UNITLESS_FULL_EMPTY", + EMPTY_FULL_UNITLESS = "EMPTY_FULL_UNITLESS", + FULL_EMPTY_UNITLESS = "FULL_EMPTY_UNITLESS", +} + +export const DEFAULT_INGREDIENT_POSTPROCESSING = [ + "TL", + "EL", + "dl", + "kl.", + "gr.", +]; + export interface Settings { /** * The number of workers to spawn concurrently when importing Recipes. * @default number of CPU cores on the system */ - importWorkerCount: number | null; + importWorkerCount: number; + + /** + * Configure in which order ingredients are sorted. + * @see IngredientSortOrder#FULL_UNITLESS_EMPTY + */ + ingredientSortOrder: IngredientSortOrder; + + /** + * The ingredient extraction parser does not work one hundred percent for all locales. + * Strings may be specified here which will be moved from an ingredient's description to its unit. + * @see DEFAULT_INGREDIENT_POSTPROCESSING + */ + ingredientUnitPostprocessing: string[]; - // TODO doc & impl - ingredientSortMode?: unknown; + /** + * The User Agent to use when doing HTTP Requests. + * @see #DEFAULT_USER_AGENT + */ + userAgent: string; } -export const DEFAULT_SETTINGS: Settings = { - importWorkerCount: null, -}; +export const DEFAULT_SETTINGS: Partial = {}; const CPU_CORES = getCpuCores(); const Schema = z.object({ importWorkerCount: z.number().refine( @@ -25,8 +70,20 @@ const Schema = z.object({ message: `importWorkerCount: Value ${val} must be greater 0 and lower than number of CPU cores available, i.e. ${getCpuCores()}.`, }), + ).optional().default(CPU_CORES || 1), + ingredientSortOrder: z.string().refine( + (val: string) => + Boolean(IngredientSortOrder[val as unknown as IngredientSortOrder]), + (val: string) => ({ + message: `ingredientSortOrder: Value ${val} must be one of ${ + Object.values(IngredientSortOrder) + }`, + }), + ).optional().default(IngredientSortOrder.FULL_UNITLESS_EMPTY), + ingredientUnitPostprocessing: z.array(z.string()).optional().default( + DEFAULT_INGREDIENT_POSTPROCESSING, ), - ingredientSortMode: z.number().optional(), + userAgent: z.string().optional().default(DEFAULT_USER_AGENT), }); export const SETTINGS_FILENAME = "settings.json"; @@ -36,7 +93,7 @@ export async function readFromDisk(configDir: string): Promise { if (await fs.exists(file)) { const contents = await Deno.readTextFile(file); try { - return Schema.parse(JSON.parse(contents)); + return Schema.parse(JSON.parse(contents)) as Settings; } catch (e) { throw new Error(`Error reading ${SETTINGS_FILENAME}: ${e}`); } @@ -46,5 +103,5 @@ export async function readFromDisk(configDir: string): Promise { JSON.stringify(DEFAULT_SETTINGS) }` ); - return DEFAULT_SETTINGS; + return Schema.parse(DEFAULT_SETTINGS) as Settings; } diff --git a/src/tpl/helpers/helpers.ts b/src/tpl/helpers/helpers.ts index 1f92404..a834d07 100644 --- a/src/tpl/helpers/helpers.ts +++ b/src/tpl/helpers/helpers.ts @@ -1,15 +1,23 @@ import { UrlHelper } from "../../http/url_helper.ts"; +import { AppState } from "../../http/webserver.ts"; import { AssetsHelper } from "./assets_helper.ts"; import { FormatHelper } from "./format_helper.ts"; import { IngredientHelper } from "./ingredient_helper.ts"; import { TranslationHelper } from "./translation_helper.ts"; -export const Helpers = { - ...AssetsHelper.INSTANCE.api, - ...TranslationHelper.INSTANCE.api, - ...IngredientHelper.INSTANCE.api, - ...FormatHelper.INSTANCE.api, - ...{ - u: UrlHelper.INSTANCE, - }, -}; +export type Helpers = AssetsHelper | TranslationHelper | IngredientHelper | FormatHelper | { u: UrlHelper }; + +export function helperFactory(appState: AppState): Helpers { + return { + ...AssetsHelper.INSTANCE.api, + ...TranslationHelper.INSTANCE.api, + ...new IngredientHelper( + appState.settings.ingredientSortOrder, + appState.settings.ingredientUnitPostprocessing, + ).api, + ...FormatHelper.INSTANCE.api, + ...{ + u: UrlHelper.INSTANCE, + }, + }; +} diff --git a/src/tpl/helpers/ingredient_helper.ts b/src/tpl/helpers/ingredient_helper.ts index f3e4e05..adc7aa6 100644 --- a/src/tpl/helpers/ingredient_helper.ts +++ b/src/tpl/helpers/ingredient_helper.ts @@ -1,4 +1,5 @@ import { parseIngredient } from "../../../deps.ts"; +import { IngredientSortOrder } from "../../settings.ts"; import { roundUpToThreeDigits } from "../../util.ts"; // copied from https://github.com/jakeboone02/parse-ingredient/blob/master/src/index.ts#L3 @@ -56,9 +57,11 @@ interface Ingredient { amountType: AmountType; } -const ingredientToWeight = ( +type IngredientWeightFunction = (i: Ingredient) => number; + +const ingredientToWeightFn = ( order: AmountType[], -): (ingredient: Ingredient) => number => { +): IngredientWeightFunction => { return (ingredient) => { for (let i = order.length - 1; i >= 0; i--) { if (ingredient.amountType === order[i]) { @@ -69,25 +72,57 @@ const ingredientToWeight = ( }; }; -const SortOrder = { - ORIGINAL: undefined, - FULL_UNITLESS_EMPTY: ingredientToWeight(["full", "unitless", "empty"]), - UNITLESS_EMPTY_FULL: ingredientToWeight(["unitless", "empty", "full"]), - EMPTY_UNITLESS_FULL: ingredientToWeight(["empty", "unitless", "full"]), - UNITLESS_FULL_EMPTY: ingredientToWeight(["unitless", "full", "empty"]), - EMPTY_FULL_UNITLESS: ingredientToWeight(["empty", "full", "unitless"]), - FULL_EMPTY_UNITLESS: ingredientToWeight(["full", "empty", "unitless"]), +const SortOrder: Record< + IngredientSortOrder, + IngredientWeightFunction | undefined +> = { + [IngredientSortOrder.ORIGINAL]: undefined, + [IngredientSortOrder.FULL_UNITLESS_EMPTY]: ingredientToWeightFn([ + "full", + "unitless", + "empty", + ]), + [IngredientSortOrder.UNITLESS_EMPTY_FULL]: ingredientToWeightFn([ + "unitless", + "empty", + "full", + ]), + [IngredientSortOrder.EMPTY_UNITLESS_FULL]: ingredientToWeightFn([ + "empty", + "unitless", + "full", + ]), + [IngredientSortOrder.UNITLESS_FULL_EMPTY]: ingredientToWeightFn([ + "unitless", + "full", + "empty", + ]), + [IngredientSortOrder.EMPTY_FULL_UNITLESS]: ingredientToWeightFn([ + "empty", + "full", + "unitless", + ]), + [IngredientSortOrder.FULL_EMPTY_UNITLESS]: ingredientToWeightFn([ + "full", + "empty", + "unitless", + ]), }; export class IngredientHelper { - public static INSTANCE: IngredientHelper = new IngredientHelper(); - + private readonly sortOrder: IngredientSortOrder; + private readonly unitPostprocessing: string[]; private cache: Map = new Map(); - private constructor() { + public constructor( + sortOrder: IngredientSortOrder, + unitPostprocessing: string[], + ) { + this.sortOrder = sortOrder; + this.unitPostprocessing = unitPostprocessing; } - private static parse(raw: string): ParsedIngredient | undefined { + private parse(raw: string): ParsedIngredient | undefined { const candidates: ParsedIngredient[] = parseIngredient(raw, { normalizeUOM: false, }); @@ -96,8 +131,7 @@ export class IngredientHelper { if (parsed) { // TODO make configurable if (!parsed.unitOfMeasure) { - const postprocessing = ["TL", "EL", "dl", "kl.", "gr."]; - const match = postprocessing.find((prefix) => + const match = this.unitPostprocessing.find((prefix) => parsed.description.startsWith(prefix) ); if (match) { @@ -125,7 +159,7 @@ export class IngredientHelper { portions = 1, ): Ingredient => { if (!this.cache.has(raw)) { - const parsed = IngredientHelper.parse(raw); + const parsed = this.parse(raw); this.cache.set(raw, parsed); } const cached = this.cache.get(raw)!; @@ -153,27 +187,19 @@ export class IngredientHelper { }; }; - public sortedIngredients( + public ingredients = ( ingredients: string[], recipeYield = 1, portions = 1, - sortOrder = SortOrder.FULL_UNITLESS_EMPTY, - ): Ingredient[] { - // TODO add configuration option for ingredient sorting + ): Ingredient[] => { + const sortFn = SortOrder[this.sortOrder]!; return ingredients .map((i) => this.ingredient(i, recipeYield, portions)) - .sort( - sortOrder - ? (i1, i2) => { - // @ts-ignore sortOrder can not be null or undefined here - return sortOrder(i1) - sortOrder(i2); - } - : undefined, - ); - } + .sort((i1, i2) => sortFn(i1) - sortFn(i2)); + }; public api = { ingredient: this.ingredient, - ingredients: this.sortedIngredients, + ingredients: this.ingredients, }; } diff --git a/src/tpl/mod.ts b/src/tpl/mod.ts index 204e040..600aceb 100644 --- a/src/tpl/mod.ts +++ b/src/tpl/mod.ts @@ -5,7 +5,7 @@ import { ImportResult } from "../data/parse/import/import_recipe.ts"; import { Eta, log, path } from "../../deps.ts"; import { AppState } from "../http/webserver.ts"; import { root } from "../util.ts"; -import { Helpers } from "./helpers/helpers.ts"; +import { helperFactory, Helpers } from "./helpers/helpers.ts"; const TEMPLATE_DIR = root("src", "tpl", "templates"); @@ -19,7 +19,7 @@ interface TemplateData { data?: Data; appState: AppState; currentUrl: URL; - h: typeof Helpers; + h: Helpers; } export class Template { @@ -68,7 +68,7 @@ export class Template { currentUrl, ...data || {}, ...{ - h: Helpers, + h: helperFactory(appState), }, }; }