Merge upstream

readme.md

@@ -14,7 +14,7 @@ You can learn the story behind the project in this [4 min video by Tim Daubensch
 
 #### Basic requirements:
 
-- node >= 16
+- node >= 19 (best to check package.json value though)
 - RPC nodes on Ethereum Mainnet & Optimism
 
 It's highly likely that you'll need either a paid Alchemy account to make the node work because it is downloading a lot of block data. You might also try rate-limiting the speed at which the node tries to download event logs.

src/api.mjs

@@ -74,7 +74,7 @@ export function handleMessage(trie, libp2p, getAllowlist, getDelegations) {
       return sendError(reply, code, httpMessage, err.toString());
     }
 
-    if (request.query.wait && request.query.wait === "true") {
+    if (request.query && request.query.wait && request.query.wait === "true") {
       await newest.recompute(trie);
       await images.recompute(trie);
     } else {

src/launch.mjs

@@ -1,5 +1,5 @@
 // @format
-import { env } from "process";
+import { env, exit } from "process";
 
 import { boot as crawl } from "@attestate/crawler";
 
@@ -58,6 +58,14 @@ const posts = await store.posts(
   allowlist,
   delegations,
 );
+try {
+  store.cache(posts);
+} catch (err) {
+  log(
+    `launch: An irrecoverable error during upvote caching occurred. "${err.toString()}`,
+  );
+  exit(1);
+}
 await newest.recompute(trie);
 await images.recompute(trie);
 karma.count(posts);

src/store.mjs

@@ -26,6 +26,23 @@ import * as messages from "./topics/messages.mjs";
 import { newWalk } from "./WalkController.mjs";
 
 const maxReaders = 500;
+
+export const upvotes = new Set();
+export function passes(marker) {
+  const exists = upvotes.has(marker);
+  if (!exists) {
+    upvotes.add(marker);
+  }
+  return !exists;
+}
+export function cache(posts) {
+  log("Caching upvote ids of posts, this can take a minute...");
+  for (const { identity, href, type } of posts) {
+    const marker = upvoteID(identity, href, type);
+    passes(marker);
+  }
+}
+
 export async function create(options) {
   return await Trie.create({
     // TODO: Understand if this should this use "resolve"? The metadata db uses
@@ -49,19 +66,6 @@ export async function create(options) {
   });
 }
 
-export function metadata(options) {
-  const db = open({
-    compression: true,
-    name: "constraints",
-    encoding: "cbor",
-    keyEncoding: "ordered-binary",
-    path: resolve(env.DATA_DIR),
-    maxReaders,
-    ...options,
-  });
-  return db;
-}
-
 // NOTE: https://ethereum.github.io/execution-specs/diffs/frontier_homestead/trie/index.html#ethereum.frontier.trie.encode_internal_node
 export function hash(node) {
   if (
@@ -240,30 +244,14 @@ export function upvoteID(identity, link, type) {
   return `${utils.getAddress(identity)}|${normalizeUrl(link)}|${type}`;
 }
 
-// TODO: The current synchronization algorithm makes use of checkpoints,
-// commits and reverts, but this function is used in sync.put and store.add,
-// but it isn't checkpointing or reverting, it just writes directly - even upon
-// soft writes into the trie. This is an issue as we e.g. could do a sync where
-// most writes are soft-written, the sync fails, but the actual constraints are
-// then written to the database.
-export async function passes(db, key, identity) {
-  // NOTE: db.doesExist seemed to have lead in some cases to a greedy match of
-  // the identifier hence returning true negatives. Meaning, it blocked users
-  // from upvoting although they had never upvoted that link.
-  // See: https://github.com/kriszyp/lmdbx-js/issues/17
-  const notFound = (await db.get(key)) === undefined;
-  await db.put(key, true);
-  return notFound;
-}
-
 export async function add(
   trie,
   message,
   libp2p,
   allowlist,
   delegations,
   synching = false,
-  metadb = metadata(),
+  metadb = upvotes,
 ) {
   const address = verify(message);
   const identity = eligible(allowlist, delegations, address);
@@ -292,7 +280,7 @@ export async function add(
   }
 
   const key = upvoteID(identity, message.href, message.type);
-  const legit = await passes(metadb, key, identity);
+  const legit = await passes(key);
   if (!legit) {
     const err = `Message "${JSON.stringify(
       message,
@@ -308,24 +296,11 @@ export async function add(
   try {
     await trie.put(Buffer.from(index, "hex"), canonical);
   } catch (err) {
-    // NOTE/TODO: Additionally, between this function and store.add (in which
-    // we use trie.put), there is no proper atomicity of storage. The trie.put
-    // function could technically crash the entire program with a fatal error
-    // and that would not be caught and hence the metadb.remove(...) statement
-    // below wouldn't be called. This can lead to real problems where the two
-    // databases, the trie and metadb, can get out of synchronization.
-    //
-    // Although lmdb-js has a method for atomically storing an entry in two
-    // sub- databases directly, it's not recommended making the transaction
-    // callback asynchronous, and so since trie.put implements a complex set of
-    // steps, it seems fragile or outright impossible to write these two values
-    // into the trie at the same time.
-    //
-    // Potentially, however, the above may also apply too high standards, as
-    // it's also unclear what lmdb-js does upon node.js panicing at a certain
-    // point in the program's execution. It may similarly corrupt the file or
-    // write without atomicity...
-    const result = await metadb.remove(key);
+    // NOTE: If trie.put crashes the program and upvotes.delete is hence not
+    // rolled back, this is not a problem as "upvotes" is only a memory-stored
+    // cached data structure that gets recomputed upon every restart of the
+    // application.
+    const result = upvotes.delete(key);
 
     let message = `trie.put failed with "${err.toString()}". Successfully rolled back constraint`;
     if (!result) {

src/sync.mjs

@@ -297,7 +297,7 @@ export async function initiate(
   );
 }
 
-export async function put(trie, message, metadb, allowlist, delegations) {
+export async function put(trie, message, allowlist, delegations) {
   let missing;
   try {
     missing = deserialize(message);
@@ -329,15 +329,7 @@ export async function put(trie, message, metadb, allowlist, delegations) {
     const libp2p = null;
     const synching = true;
     try {
-      await store.add(
-        trie,
-        obj,
-        libp2p,
-        allowlist,
-        delegations,
-        synching,
-        metadb,
-      );
+      await store.add(trie, obj, libp2p, allowlist, delegations, synching);
       log(`Adding to database value (as JSON)`);
     } catch (err) {
       // NOTE: We're not bubbling the error up here because we want to be
@@ -444,25 +436,16 @@ export function handleLeaves(trie, peerFab) {
     log("handleLeaves: Received leaves and storing them in db");
 
     try {
-      const metadb = store.metadata();
-      await metadb.transaction(async () => {
-        try {
-          // NOTE: We're adding multiple statements here to the try catch
-          // because in each of their failure, we want to abort writing into
-          // the databases.
-          const allowlist = await registry.allowlist();
-          const delegations = await registry.delegations();
-          await put(trie, message, metadb, allowlist, delegations);
-          return true;
-        } catch (err) {
-          elog(err, "handleLeaves: Unexpected error");
-          await trie.revert();
-          peerFab.set();
-          return false;
-        }
-      });
+      // NOTE: We're adding multiple statements here to the try catch
+      // because in each of their failure, we want to abort writing into
+      // the databases.
+      const allowlist = await registry.allowlist();
+      const delegations = await registry.delegations();
+      await put(trie, message, allowlist, delegations);
     } catch (err) {
-      elog(err, "error in metadb callback");
+      elog(err, "handleLeaves: Unexpected error");
+      await trie.revert();
+      peerFab.set();
     }
 
     // NOTE: While there could be a strategy where we continuously stay in a

src/views/components/footer.mjs

@@ -123,6 +123,8 @@ const footer = (theme, path) => html`
     <span> | </span>
     <a href="/kiwipass">Kiwi Pass</a>
     <span> | </span>
+    <a target="_blank" href="https://github.com/attestate/kiwistand">Source code</a>
+    <span> | </span>
     <a href="https://dune.com/rvolz/kiwi-news" target"_blank">Dune Dashboard</a>
     <span> | </span>
     <a

src/views/components/row.mjs

@@ -105,7 +105,7 @@ const row = (
                   ? html`<br /><a target="_blank" href="${story.href}"
                         ><img
                           loading="lazy"
-                          style="max-width: 80vw; padding: 0.75rem 1rem 0 0; max-height: 30vh"
+                          style="max-width: 80vw; padding: 0.75rem 1rem 0 0; max-height: 20vh"
                           src="${story.image}"
                       /></a>`
                   : ""}

src/views/story.mjs

@@ -132,6 +132,10 @@ export default async function (trie, theme, index, value, identity) {
     (a, b) => a.timestamp - b.timestamp,
   );
   story.avatars = avatars;
+  // NOTE: store.post returns upvoters as objects of "identity" and "timestamp"
+  // property so that we can zip them with tipping actions. But the row component
+  // expects upvoters to be a string array of Ethereum addresses.
+  story.upvoters = story.upvoters.map(({ identity }) => identity);
 
   const ensData = await ens.resolve(story.identity);
   story.submitter = ensData;

src/web/src/Tip.jsx

@@ -1,12 +1,21 @@
 // @format
+import { RainbowKitProvider } from "@rainbow-me/rainbowkit";
+import { WagmiConfig, useAccount } from "wagmi";
 import { PayKitProvider } from "@dawnpay/kit";
 import { useDawnPay } from "@dawnpay/kit";
 
+import { client, chains } from "./client.mjs";
+import { getLocalAccount } from "./session.mjs";
+
 const Container = (props) => {
   return (
-    <PayKitProvider>
-      <Tip {...props} />
-    </PayKitProvider>
+    <WagmiConfig client={client}>
+      <RainbowKitProvider chains={chains}>
+        <PayKitProvider>
+          <Tip {...props} />
+        </PayKitProvider>
+      </RainbowKitProvider>
+    </WagmiConfig>
   );
 };
 
@@ -18,6 +27,17 @@ const Tip = (props) => {
     await pay(props.address, props.metadata);
   };
 
+  let address;
+  const account = useAccount();
+  const localAccount = getLocalAccount(account.address);
+  if (account.isConnected) {
+    address = account.address;
+  }
+  if (localAccount) {
+    address = localAccount.identity;
+  }
+  if (props.address === address) return;
+
   return (
     <span>
       <span> • </span>

test/api_test.mjs

@@ -115,7 +115,7 @@ test("listMessages success", async (t) => {
   const response = await listMessages(
     trie,
     getAllowlist,
-    getDelegations
+    getDelegations,
   )(mockRequest, mockReply);
 
   console.log(response);
@@ -172,6 +172,7 @@ test("handleMessage should send back an error upon invalid address signer", asyn
 });
 
 test("handleMessage should handle a valid message and return 200 OK", async (t) => {
+  store.upvotes.clear();
   const address = "0x0f6A79A579658E401E0B81c6dde1F2cd51d97176";
   const privateKey =
     "0xad54bdeade5537fb0a553190159783e45d02d316a992db05cbed606d3ca36b39";
@@ -187,6 +188,9 @@ test("handleMessage should handle a valid message and return 200 OK", async (t)
 
   const request = {
     body: signedMessage,
+    query: {
+      wait: false,
+    },
   };
 
   const reply = {

test/dotenv_test.mjs

@@ -26,23 +26,25 @@ test("that repo contains a .env-copy file with all possible configuration option
   const expr = new RegExp(".*=.*", "gm");
   const allOptions = [
     "OPTIMISM_RPC_HTTP_HOST",
-    "LOG_LEVEL",
+    "RPC_HTTP_HOST",
+    "DEBUG",
     "NODE_ENV",
     "BIND_ADDRESS_V4",
     "PORT",
     "IS_BOOTSTRAP_NODE",
     "USE_EPHEMERAL_ID",
     "IPV4",
-    "HTTP_PORT",
-    "API_PORT",
     "HTTP_MESSAGES_MAX_PAGE_SIZE",
     "DATA_DIR",
     "CACHE_DIR",
     "AUTO_SYNC",
     "ROOT_ADVERTISEMENT_TIMEOUT",
     "MIN_TIMESTAMP_SECS",
-    "TIMESTAMP_TOLERANCE_SECS",
-    "DEBUG",
+    "MAX_TIMESTAMP_DELTA_SECS",
+    "HTTP_PORT",
+    "API_PORT",
+    "TOTAL_STORIES",
+    "TOTAL_USERS",
   ];
   await access(envPath, constants.F_OK);
   const envContent = (await readFile(envPath)).toString();
@@ -56,6 +58,6 @@ test("that repo contains a .env-copy file with all possible configuration option
   t.is(
     copyMatches.length,
     allOptions.length,
-    ".env-copy and required `allOptions` mismatch",
+    `.env-copy and required "allOptions" mismatch, copyMatches: "${copyMatches}" and allOptions: "${allOptions}"`,
   );
 });