squash snapshot workflow commits

.github/workflows/ci.yml

@@ -3,7 +3,9 @@ name: Dev CI/CD
 on:
   push:
     branches:
-      - '*'
+      - 'master'
+  pull_request:
+    types: [opened]
 
 jobs:
   build:

.github/workflows/weekly-snapsots.yml

@@ -0,0 +1,70 @@
+name: Weekly Sentinel Snapshots
+on:
+  schedule:
+    - cron: '0 0 * * 0'
+  push:
+    branches:
+      - '*'
+
+jobs:
+  snapshots:
+    name: "Sentinel Snapshots"
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        network: [base-mainnet, bsc-mainnet, eth-mainnet, optimism-mainnet]
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Get current date
+        id: get-date
+        run: echo "{date}=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20.x
+
+      - name: Install jq
+        uses: dcarbone/[email protected]
+
+      - name: Set up IPFS
+        uses: ibnesayeed/setup-ipfs@master
+        id: ipfs_setup
+
+      - name: Generate networks file
+        run: ./scripts/getSFMetadata.sh
+
+      - name: Build Snapshot
+        run: ./scripts/manageSnapshots.sh -g ${{ matrix.network }} > ${{ matrix.network }}.txt
+
+      - name: Upload Snapshot
+        run: ./scripts/manageSnapshots.sh -u
+
+      - name: Collate Changes
+        run: |
+          #!/bin/bash
+          if [[ ! -f "manifest.json" || ! -f "manifest-tmp.json" ]]; then
+            echo "One or both files do not exist."
+            exit 0
+          fi
+
+          if ! diff -q manifest.json manifest-tmp.json > /dev/null; then
+            echo "Files are different. Outputting manifest-tmp.json."
+            cat manifest-tmp.json
+          else
+            echo "Files are identical. No output needed."
+          fi
+
+      - name: Create Pull Request
+        uses: peter-evans/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          add-paths: |
+            manifest.json
+          commit-message: ${{ steps.get-date.outputs.date }} uupdate manifest
+          title: 'Update manifests for ${{ steps.get-date.outputs.date }}'

.gitignore

@@ -5,6 +5,8 @@ datadir/
 snapshots/
 coverage
 typechain
+networks
+*error.log
 
 # Hardhat files
 cache

scripts/buildSnapshot.js

@@ -15,7 +15,7 @@ const EventModel = require("./../src/models/EventModel");
 const Bootstrap = require("./../src/boot/bootstrap");
 const LoadEvents = require("./../src/boot/loadEvents");
 const DB = require("./../src/database/db");
-const Repository = require("./../src/database/repository");
+const Repository = require("./../src/database/businessRepository");
 const Timer = require("./../src/utils/timer");
 const metadata = require("@superfluid-finance/metadata/networks.json");
 const {QueryTypes} = require("sequelize");

scripts/generateManifest.js

@@ -0,0 +1,54 @@
+const { writeFileSync, readFileSync } = require("fs");
+
+function cleanLogData(path) {
+    const newCIDs = new Map();
+    try {
+        readFileSync(path, "utf8").split(/\r?\n/).forEach(line => {
+            const splitLine = line.split(",");
+            const fileName = splitLine[0];
+            const filtered = fileName.match("_(.*)_")
+            if(filtered) {
+                newCIDs.set(filtered[1], splitLine[1]);
+            }
+        });
+        return newCIDs;
+    } catch (err) {
+        console.log(err);
+    }
+}
+
+(() => {
+    try {
+        const myArgs = process.argv.slice(2);
+        const ipfsLog = myArgs[0];
+        const outputFile = myArgs[1];
+
+        console.log(`ipfs log file: ${ipfsLog}, output file: ${outputFile}`);
+
+        if(!ipfsLog) {
+            throw new Error("No IPFS log")
+        }
+        if(!outputFile) {
+            throw new Error("No output file")
+        }
+
+        const newCIDs = cleanLogData(ipfsLog);
+        /*if(newCIDs.size !== 10) {
+            throw new Error("IPFS log not complety")
+        }*/
+
+        // Read manifest data from local file
+        const manifestJson = JSON.parse(readFileSync('manifest.json', 'utf8'));
+
+        // Update manifest in memory
+        for (const [key, value] of newCIDs) {
+            manifestJson.networks[key].cid = value;
+        }
+
+        // Write updated manifest to output file
+        writeFileSync(outputFile, JSON.stringify(manifestJson, null, 2));
+
+    } catch (err) {
+        console.error(err)
+    }
+})();

scripts/getSFMetadata.sh

@@ -0,0 +1,29 @@
+#Generate networks file from Superfluid Metadata
+#!/bin/bash
+
+# Superfluid Metdata URL
+input_json_url="https://raw.githubusercontent.com/superfluid-finance/protocol-monorepo/dev/packages/metadata/networks.json"
+
+# Variables
+#TODO: Move variable to github secret
+common_domain="${COMMON_DOMAIN:-.rpc.x.superfluid.dev}"
+
+# Download the input JSON file
+wget -O input.json "$input_json_url"
+
+# Check if jq is installed
+if ! command -v jq &>/dev/null; then
+    echo "jq is required but it's not installed. Aborting."
+    exit 1
+fi
+
+# Process the input JSON file
+# Create output file
+output_file="output.txt"
+
+# Extract data from JSON and write to text file
+jq -r --arg common_domain "$common_domain" '.[] | "\(.name),https://\(.name)\($common_domain)"' input.json > networks
+
+echo "Text file created: networks"
+
+rm input.json

scripts/manageSnapshots.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+set -xe
+
+#Variables
+filename="networks"
+ipfs_api="${IPFS_API:-/ip4/65.21.152.182/tcp/5001}"
+
+generate_snapshot() {
+    echo "Generating new snapshots..."
+    yarn install
+
+    if [ ! -d "snapshots" ]; then
+        mkdir snapshots
+    fi
+
+    # Check if network argument is provided
+    if [ -z "$1" ]; then
+        echo "Usage: $0 -g <network>"
+        exit 1
+    fi
+
+    # Search for the specified network in the filename
+    if grep -q "^$1," "$filename"; then
+        echo "Generating snapshot for $1..."
+        url=$(grep "^$1," "$filename" | cut -d ',' -f 2)  # Extract URL
+        [ -n "$url" ] && node ./scripts/buildSnapshot.js "$url"
+        echo "Generating done"
+    else
+        echo "Error: Network '$1' not found in '$filename'."
+        exit 1
+    fi
+}
+
+
+upload_snapshot() {
+    echo "Uploading snapshots..."
+    ipfs_logfile="logs/ipfs_$(date '+%Y-%m-%d').txt"
+    rm -f -- "$ipfs_logfile"
+    for file in ./snapshots/*.sqlite.gz; do
+      ipfs_hash=`ipfs --api "$ipfs_api" add -q $file`
+      echo $file,$ipfs_hash >> "$ipfs_logfile"
+    done
+    rm manifest.json
+    node ./scripts/generateManifest.js "$ipfs_logfile" manifest.json
+    ipfs_hash=`ipfs --api "$ipfs_api" add -q manifest.json`
+    echo manifest.json,$ipfs_hash >> "$ipfs_logfile"
+    # updating the manifest ipns link
+    ipfs --api "$ipfs_api" name publish --key=sentinel-manifest "$ipfs_hash"
+    echo "Uploading snapshots done"
+}
+
+clean_snapshots() {
+    echo "Cleaning snapshot folder..."
+    rm -f -- "$HOME/snapshots"/*.gz
+    echo "Cleaning done"
+}
+
+# Usage
+usage() {
+    echo "Usage: $0 [-g] [-u] [-p] [-c]"
+    echo "Options:"
+    echo "  -g <network-name> Generate snapshots"
+    echo "  -u    Upload snapshots"
+    echo "  -c    Clean snapshots"
+    exit 1
+}
+
+# Command line options
+while getopts "g:upc" opt; do
+    case $opt in
+        g) generate_snapshot "$OPTARG" ;;
+        u) upload_snapshot ;;
+        c) clean_snapshots ;;
+        *) usage ;;
+    esac
+done
+
+
+# If no options are provided, show usage
+if [[ $# -eq 0 ]]; then
+    usage
+fi