diff --git a/src/parser.mjs b/src/parser.mjs index 0bfedab..b36182e 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -9,6 +9,8 @@ import { JSDOM } from "jsdom"; import { fetchBuilder, FileSystemCache } from "node-fetch-cache"; import { useAgent } from "request-filtering-agent"; +import cache from "./cache.mjs"; + const fetch = fetchBuilder.withCache( new FileSystemCache({ cacheDirectory: path.resolve(env.CACHE_DIR), @@ -50,13 +52,24 @@ async function extractCanonicalLink(html) { } export const metadata = async (url) => { - const response = await fetch(url, { - agent: useAgent(url), - }); + let result, html; + if (cache.has(url)) { + const fromCache = cache.get(url); + result = fromCache.result; + html = fromCache.html; + } else { + const response = await fetch(url, { + agent: useAgent(url), + }); - const html = await response.text(); - const parsed = await ogs({ html }); - const result = parsed.result; + const html = await response.text(); + const parsed = await ogs({ html }); + result = parsed.result; + + if (result && html) { + cache.set(url, { result, html }); + } + } const domain = safeExtractDomain(url); if (filtered.includes(domain) || (result && !result.success)) { diff --git a/src/views/feed.mjs b/src/views/feed.mjs index 99b432c..7dfc217 100644 --- a/src/views/feed.mjs +++ b/src/views/feed.mjs @@ -837,7 +837,8 @@ export default async function (trie, theme, page, domain) { - ${contestStories.map( + ${contestStories && + contestStories.map( Row( start, "/", @@ -854,7 +855,7 @@ export default async function (trie, theme, page, domain) { .map( Row(start, "/", undefined, null, null, null, recentJoiners), )} - ${Row(start, "/", "", null, null, null, recentJoiners)(ad)} + ${ad && Row(start, "/", "", null, null, null, recentJoiners)(ad)} ${stories .slice(3, 8) .map(