From e4a18fb53b76778fedd8fd0f430e9537467b3ef4 Mon Sep 17 00:00:00 2001 From: mmd-afegbua Date: Tue, 20 Feb 2024 17:00:24 +0100 Subject: [PATCH] test workflow --- .github/workflows/weekly-snapsots.yml | 51 ++++++++++++++++ .gitignore | 2 + scripts/buildSnapshot.js | 2 +- scripts/generateManifest.js | 54 +++++++++++++++++ scripts/getMetadata.sh | 28 +++++++++ scripts/manageSnapshots.sh | 83 +++++++++++++++++++++++++++ 6 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/weekly-snapsots.yml create mode 100644 scripts/generateManifest.js create mode 100755 scripts/getMetadata.sh create mode 100755 scripts/manageSnapshots.sh diff --git a/.github/workflows/weekly-snapsots.yml b/.github/workflows/weekly-snapsots.yml new file mode 100644 index 00000000..9ba4f04c --- /dev/null +++ b/.github/workflows/weekly-snapsots.yml @@ -0,0 +1,51 @@ +name: Weekly Sentinel Snapshots +on: + schedule: + - cron: '0 0 * * 0' + push: + branches: + - '*' + +jobs: + snapshots: + name: "Create Snapshots, Upload and Update Manifests" + runs-on: ubuntu-latest + + strategy: + matrix: + network: [base-mainnet, bsc-mainnet, eth-mainnet, optimism-mainnet] + + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Get current date + id: get-date + run: echo "::set-output name=date::$(date +'%Y-%m-%d')" + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: 21.x + + - name: Install jq + uses: dcarbone/install-jq-action@v2.1.0 + + - name: Set up IPFS + uses: ibnesayeed/setup-ipfs@master + id: ipfs_setup + + - name: Build Snapshot + run: ./scripts/manageSnapshot.sh -g ${{ matrix.network }} + + - name: Upload Snapshot + run: ./scripts/manageSnapshot.sh -u + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v6.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + add-paths: | + manifest.json + commit-message: ${{ steps.get-date.outputs.date }} uupdate manifest + title: 'Update manifests for ${{ steps.get-date.outputs.date }}' diff --git a/.gitignore b/.gitignore index e976277d..8affd456 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ datadir/ snapshots/ coverage typechain +networks +*error.log # Hardhat files cache diff --git a/scripts/buildSnapshot.js b/scripts/buildSnapshot.js index 21888c64..40177435 100644 --- a/scripts/buildSnapshot.js +++ b/scripts/buildSnapshot.js @@ -15,7 +15,7 @@ const EventModel = require("./../src/models/EventModel"); const Bootstrap = require("./../src/boot/bootstrap"); const LoadEvents = require("./../src/boot/loadEvents"); const DB = require("./../src/database/db"); -const Repository = require("./../src/database/repository"); +const Repository = require("./../src/database/businessRepository"); const Timer = require("./../src/utils/timer"); const metadata = require("@superfluid-finance/metadata/networks.json"); const {QueryTypes} = require("sequelize"); diff --git a/scripts/generateManifest.js b/scripts/generateManifest.js new file mode 100644 index 00000000..309f1ecf --- /dev/null +++ b/scripts/generateManifest.js @@ -0,0 +1,54 @@ +const { writeFileSync, readFileSync } = require("fs"); + +function cleanLogData(path) { + const newCIDs = new Map(); + try { + readFileSync(path, "utf8").split(/\r?\n/).forEach(line => { + const splitLine = line.split(","); + const fileName = splitLine[0]; + const filtered = fileName.match("_(.*)_") + if(filtered) { + newCIDs.set(filtered[1], splitLine[1]); + } + }); + return newCIDs; + } catch (err) { + console.log(err); + } +} + +(() => { + try { + const myArgs = process.argv.slice(2); + const ipfsLog = myArgs[0]; + const outputFile = myArgs[1]; + + console.log(`ipfs log file: ${ipfsLog}, output file: ${outputFile}`); + + if(!ipfsLog) { + throw new Error("No IPFS log") + } + if(!outputFile) { + throw new Error("No output file") + } + + const newCIDs = cleanLogData(ipfsLog); + /*if(newCIDs.size !== 10) { + throw new Error("IPFS log not complety") + }*/ + + // Read manifest data from local file + const manifestJson = JSON.parse(readFileSync('manifest.json', 'utf8')); + + // Update manifest in memory + for (const [key, value] of newCIDs) { + manifestJson.networks[key].cid = value; + } + + // Write updated manifest to output file + writeFileSync(outputFile, JSON.stringify(manifestJson, null, 2)); + + } catch (err) { + console.error(err) + } +})(); diff --git a/scripts/getMetadata.sh b/scripts/getMetadata.sh new file mode 100755 index 00000000..235ce389 --- /dev/null +++ b/scripts/getMetadata.sh @@ -0,0 +1,28 @@ +#Generate networks file from Superfluid Metadata +#!/bin/bash + +# Superfluid Metdata URL +input_json_url="https://raw.githubusercontent.com/superfluid-finance/protocol-monorepo/dev/packages/metadata/networks.json" + +# Variables +common_domain="${COMMON_DOMAIN}" + +# Download the input JSON file +wget -O input.json "$input_json_url" + +# Check if jq is installed +if ! command -v jq &>/dev/null; then + echo "jq is required but it's not installed. Aborting." + exit 1 +fi + +# Process the input JSON file +# Create output file +output_file="output.txt" + +# Extract data from JSON and write to text file +jq -r --arg common_domain "$common_domain" '.[] | "\(.name),https://\(.name)\($common_domain)"' input.json > networks + +echo "Text file created: networks" + +rm input.json diff --git a/scripts/manageSnapshots.sh b/scripts/manageSnapshots.sh new file mode 100755 index 00000000..0e7dd7c1 --- /dev/null +++ b/scripts/manageSnapshots.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +set -xe + +#Variables +filename="networks" +ipfs_api="${IPFS_API:-"/ip4/65.21.152.182/tcp/5001"}" + +generate_snapshot() { + echo "Generating new snapshots..." + yarn install + + if [ ! -d "snapshots" ]; then + mkdir snapshots + fi + + # Check if network argument is provided + if [ -z "$1" ]; then + echo "Usage: $0 -g " + exit 1 + fi + + # Search for the specified network in the filename + if grep -q "^$1," "$filename"; then + echo "Generating snapshot for $1..." + url=$(grep "^$1," "$filename" | cut -d ',' -f 2) # Extract URL + [ -n "$url" ] && node ./scripts/buildSnapshot.js "$url" + echo "Generating done" + else + echo "Error: Network '$1' not found in '$filename'." + exit 1 + fi +} + + +upload_snapshot() { + echo "Uploading snapshots..." + ipfs_logfile="logs/ipfs_$(date '+%Y-%m-%d').txt" + rm -f -- "$ipfs_logfile" + for file in ./snapshots/*.sqlite.gz; do + ipfs_hash=`ipfs --api "$ipfs_api" add -q $file` + echo $file,$ipfs_hash >> "$ipfs_logfile" + done + rm manifest.json + node ./scripts/generateManifest.js "$ipfs_logfile" manifest.json + ipfs_hash=`ipfs --api "$ipfs_api" add -q manifest.json` + echo manifest.json,$ipfs_hash >> "$ipfs_logfile" + # updating the manifest ipns link + ipfs --api "$ipfs_api" name publish --key=sentinel-manifest "$ipfs_hash" + echo "Uploading snapshots done" +} + +clean_snapshots() { + echo "Cleaning snapshot folder..." + rm -f -- "$HOME/snapshots"/*.gz + echo "Cleaning done" +} + +# Usage +usage() { + echo "Usage: $0 [-g] [-u] [-p] [-c]" + echo "Options:" + echo " -g Generate snapshots" + echo " -u Upload snapshots" + echo " -c Clean snapshots" + exit 1 +} + +# Command line options +while getopts "g:upc" opt; do + case $opt in + g) generate_snapshot "$OPTARG" ;; + u) upload_snapshot ;; + c) clean_snapshots ;; + *) usage ;; + esac +done + + +# If no options are provided, show usage +if [[ $# -eq 0 ]]; then + usage +fi