Add parser cache back

attestate · Oct 23, 2024 · 7828161 · 7828161
1 parent 1af067f
commit 7828161
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 8 deletions.
diff --git a/src/parser.mjs b/src/parser.mjs
@@ -9,6 +9,8 @@ import { JSDOM } from "jsdom";
 import { fetchBuilder, FileSystemCache } from "node-fetch-cache";
 import { useAgent } from "request-filtering-agent";
 
+import cache from "./cache.mjs";
+
 const fetch = fetchBuilder.withCache(
   new FileSystemCache({
     cacheDirectory: path.resolve(env.CACHE_DIR),
@@ -50,13 +52,24 @@ async function extractCanonicalLink(html) {
 }
 
 export const metadata = async (url) => {
-  const response = await fetch(url, {
-    agent: useAgent(url),
-  });
+  let result, html;
+  if (cache.has(url)) {
+    const fromCache = cache.get(url);
+    result = fromCache.result;
+    html = fromCache.html;
+  } else {
+    const response = await fetch(url, {
+      agent: useAgent(url),
+    });
 
-  const html = await response.text();
-  const parsed = await ogs({ html });
-  const result = parsed.result;
+    const html = await response.text();
+    const parsed = await ogs({ html });
+    result = parsed.result;
+
+    if (result && html) {
+      cache.set(url, { result, html });
+    }
+  }
 
   const domain = safeExtractDomain(url);
   if (filtered.includes(domain) || (result && !result.success)) {

diff --git a/src/views/feed.mjs b/src/views/feed.mjs
@@ -837,7 +837,8 @@ export default async function (trie, theme, page, domain) {
                   </div>
                 </td>
               </tr>
-              ${contestStories.map(
+              ${contestStories &&
+              contestStories.map(
                 Row(
                   start,
                   "/",
@@ -854,7 +855,7 @@ export default async function (trie, theme, page, domain) {
                 .map(
                   Row(start, "/", undefined, null, null, null, recentJoiners),
                 )}
-              ${Row(start, "/", "", null, null, null, recentJoiners)(ad)}
+              ${ad && Row(start, "/", "", null, null, null, recentJoiners)(ad)}
               ${stories
                 .slice(3, 8)
                 .map(