Initial CI workflow (#25)

Co-authored-by: Harry Solovay <[email protected]>

.github/workflows/checks.yaml

@@ -0,0 +1,97 @@
+name: checks
+on:
+  pull_request:
+  merge_group:
+  push:
+    branches:
+      - main
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dsherret/rust-toolchain-file@v1
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-check-${{ hashFiles('**/Cargo.lock') }}
+      - run: RUSTFLAGS="-D warnings" cargo check --all-targets
+
+  clippy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dsherret/rust-toolchain-file@v1
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-clippy-${{ hashFiles('**/Cargo.lock') }}
+      - run: RUSTFLAGS="-D warnings" cargo clippy --all-targets
+
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dsherret/rust-toolchain-file@v1
+      - uses: extractions/setup-just@v2
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-test-${{ hashFiles('**/Cargo.lock') }}
+      - name: Setup
+        run: |
+          just get-mainnet-archive-db
+          just pg
+          just wait-for-pg
+      - name: Test
+        run: just test
+      - name: Tear down
+        run: just pg-down
+
+  rustfmt:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dsherret/rust-toolchain-file@v1
+      - run: cargo fmt --check
+
+  dprint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-dprint
+      - run: npm i -g sql-formatter
+      - uses: actions/checkout@v4
+      - uses: dprint/[email protected]
+
+  cspell:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: streetsidesoftware/cspell-action@v6
+        with:
+          incremental_files_only: false

Justfile

@@ -1,2 +1,20 @@
 pg:
   docker run -d --name mina-archive-db -p 5432:5432 -v $(pwd)/sql_scripts:/docker-entrypoint-initdb.d -e POSTGRES_PASSWORD=whatever -e POSTGRES_USER=mina postgres
+
+pg-up:
+  docker start mina-archive-db
+
+pg-down:
+  docker kill mina-archive-db
+
+pg-rm:
+  docker rm mina-archive-db
+
+get-mainnet-archive-db:
+  ./scripts/get_archive_db.sh mainnet
+
+wait-for-pg:
+  ./scripts/wait_for_pg.sh
+
+test:
+  SNAP_CHECK=1 cargo test

README.md

@@ -1,5 +1,63 @@
 # Mina Mesh
 
-An implementation of
-[the Coinbase Mesh specification](https://docs.cdp.coinbase.com/mesh/docs/welcome) for the
-[Mina](https://minaprotocol.com/) blockchain.
+[![checks](https://github.com/MinaFoundation/MinaMesh/actions/workflows/checks.yaml/badge.svg)](https://github.com/MinaFoundation/MinaMesh/actions/workflows/checks.yaml)
+
+## Overview
+
+Mina Mesh is an implementation of the
+[Coinbase Mesh specification](https://docs.cdp.coinbase.com/mesh/docs/welcome) for the
+[Mina blockchain](https://minaprotocol.com/).
+
+## Building
+
+To build the project:
+
+```bash
+cargo build
+```
+
+The binary will be available at:
+
+```bash
+target/debug/mina_mesh
+```
+
+## Testing
+
+### Setup PostgreSQL with Latest Mainnet Archive DB
+
+To set up the testing environment with a working PostgreSQL database, use the predefined `just`
+steps:
+
+```bash
+just get-mainnet-archive-db
+just pg
+just wait-for-pg
+```
+
+> Note: This process sets up the environment using the latest mainnet archive database.
+
+### Run Tests
+
+Once the setup is complete, run the tests with:
+
+```bash
+just test
+```
+
+### Managing PostgreSQL
+
+- **Stop PostgreSQL**: To stop the PostgreSQL instance:
+
+  ```bash
+  just pg-down
+  ```
+
+- **Restart PostgreSQL**: To restart without reinitializing the database (useful if the database is
+  already set up):
+
+  ```bash
+  just pg-up
+  ```
+
+> You only need to reinitialize the database if you want the latest data dump.

cspell.json

@@ -8,6 +8,14 @@
     }
   ],
   "enableGlobDot": true,
-  "dictionaries": ["project-words"],
-  "ignorePaths": [".git", ".sqlx", "mesh-generated", "target"]
+  "dictionaries": ["project-words", "rust"],
+  "ignorePaths": [
+    ".git/**",
+    ".sqlx/**",
+    "mesh_generated/**",
+    "src/graphql/schema/mina_schema.graphql",
+    "sql/**",
+    "target/**"
+  ],
+  "useGitignore": true
 }

dprint.json

@@ -17,9 +17,9 @@
   "plugins": [
     "https://plugins.dprint.dev/dockerfile-0.3.2.wasm",
     "https://plugins.dprint.dev/exec-0.5.0.json@8d9972eee71fa1590e04873540421f3eda7674d0f1aae3d7c788615e7b7413d0",
-    "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.4.0.wasm",
-    "https://plugins.dprint.dev/json-0.19.2.wasm",
-    "https://plugins.dprint.dev/markdown-0.16.4.wasm",
+    "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.5.0.wasm",
+    "https://plugins.dprint.dev/json-0.19.3.wasm",
+    "https://plugins.dprint.dev/markdown-0.17.8.wasm",
     "https://plugins.dprint.dev/toml-0.6.2.wasm"
   ]
 }

rustfmt.toml

@@ -1,14 +1,14 @@
 edition = "2021"
 group_imports = "StdExternalCrate"
-ignore = ["mesh-generated"]
+ignore = ["mesh_generated/src/**"]
 imports_granularity = "Crate"
 max_width = 120
 newline_style = "Unix"
 overflow_delimited_expr = true
 spaces_around_ranges = true
+style_edition = "2021"
 tab_spaces = 2
 use_field_init_shorthand = true
 use_small_heuristics = "Max"
 use_try_shorthand = true
-version = "Two"
 wrap_comments = true

scripts/get_archive_db.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# This script is used to download the archive dump from the mina-archive-dumps bucket
+# and extract the archive dump to the sql_scripts directory
+# The script will download the archive dump for the last 5 days and extract the first available archive dump
+# Usage: ./scripts/get_archive_db.sh <MINA_NETWORK>
+# Example: ./scripts/get_archive_db.sh mainnet
+
+MINA_NETWORK=${1}
+MINA_ARCHIVE_DUMP_URL=${MINA_ARCHIVE_DUMP_URL:=https://storage.googleapis.com/mina-archive-dumps}
+DUMP_TIME=0000
+SQL_SCRIPT_PATH=$(pwd)/sql_scripts
+TAR_FILE_PATH=${SQL_SCRIPT_PATH}/o1labs-archive-dump.tar.gz
+
+mkdir -p ${SQL_SCRIPT_PATH}
+
+MAX_DAYS_LOOKBACK=5
+i=0
+while [ $i -lt $MAX_DAYS_LOOKBACK ]; do
+    DATE=$(date -d "$i days ago" +%G-%m-%d)_${DUMP_TIME}
+    STATUS_CODE=$(curl -s -o /dev/null --head -w "%{http_code}" "${MINA_ARCHIVE_DUMP_URL}/${MINA_NETWORK}-archive-dump-${DATE}.sql.tar.gz")
+    if [[ ! $STATUS_CODE =~ 2[0-9]{2} ]]; then
+        i=$((i + 1))
+    else
+        echo "Download ${MINA_NETWORK}-archive-dump-${DATE}.sql.tar.gz"
+        curl "${MINA_ARCHIVE_DUMP_URL}/${MINA_NETWORK}-archive-dump-${DATE}.sql.tar.gz" -o ${TAR_FILE_PATH}
+        break
+    fi
+done
+
+[[ $STATUS_CODE =~ 2[0-9]{2} ]] || echo "[WARN] Unable to find archive dump for ${MINA_NETWORK}"
+
+tar -xvf ${SQL_SCRIPT_PATH}/o1labs-archive-dump.tar.gz -C ${SQL_SCRIPT_PATH}
+rm -f ${TAR_FILE_PATH}
+
+echo "Extracted ${MINA_NETWORK}-archive-dump-${DATE}.sql.tar.gz to ${SQL_SCRIPT_PATH}/${MINA_NETWORK}-archive-dump-${DATE}.sql"

scripts/wait_for_pg.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# This script is used to check if PostgreSQL is available
+# Usage: ./scripts/pg_ready.sh <PG_HOST> <PG_PORT>
+# Example: ./scripts/pg_ready.sh localhost 5432
+
+# Parameters
+PG_HOST="${1:-localhost}"
+PG_PORT="${2:-5432}"
+
+# Wait for PostgreSQL to become available
+until pg_isready -h "$PG_HOST" -p "$PG_PORT"; do
+  echo "Waiting for PostgreSQL to become available at ${PG_HOST}:${PG_PORT}..."
+  sleep 1
+done
+
+echo "PostgreSQL is available at ${PG_HOST}:${PG_PORT}"

src/api/account_balance.rs

@@ -5,9 +5,9 @@ pub use mesh::models::{
 };
 
 use crate::{
-  MinaMesh, MinaMeshError,
   graphql::{Account, AnnotatedBalance, Balance, Length, QueryBalance, QueryBalanceVariables, StateHash},
   util::Wrapper,
+  MinaMesh, MinaMeshError,
 };
 
 /// https://github.com/MinaProtocol/mina/blob/985eda49bdfabc046ef9001d3c406e688bc7ec45/src/app/rosetta/lib/account.ml#L11

src/api/block.rs

@@ -3,8 +3,8 @@ use cynic::QueryBuilder;
 pub use mesh::models::{BlockRequest, BlockResponse, PartialBlockIdentifier};
 
 use crate::{
-  MinaMesh, MinaMeshError, Wrapper,
   graphql::{QueryBlockTransactions, QueryBlockTransactionsVariables},
+  MinaMesh, MinaMeshError, Wrapper,
 };
 
 #[derive(sqlx::Type, Debug, PartialEq, Eq)]

src/api/mempool.rs

@@ -6,7 +6,7 @@ use anyhow::Result;
 use cynic::QueryBuilder;
 pub use mesh::models::{MempoolResponse, TransactionIdentifier};
 
-use crate::{MinaMesh, graphql::QueryMempool};
+use crate::{graphql::QueryMempool, MinaMesh};
 
 /// https://github.com/MinaProtocol/mina/blob/985eda49bdfabc046ef9001d3c406e688bc7ec45/src/app/rosetta/lib/mempool.ml#L56
 impl MinaMesh {

src/api/mempool_transaction.rs

@@ -2,8 +2,8 @@ use cynic::QueryBuilder;
 pub use mesh::models::{MempoolTransactionRequest, MempoolTransactionResponse, Transaction, TransactionIdentifier};
 
 use crate::{
-  MinaMesh, MinaMeshError,
   graphql::{QueryMempoolTransactions, QueryMempoolTransactionsVariables},
+  MinaMesh, MinaMeshError,
 };
 
 /// https://github.com/MinaProtocol/mina/blob/985eda49bdfabc046ef9001d3c406e688bc7ec45/src/app/rosetta/lib/mempool.ml#L137

src/api/network_health_check.rs

@@ -2,7 +2,7 @@ use anyhow::Result;
 use cynic::QueryBuilder;
 use mesh::models::NetworkIdentifier;
 
-use crate::{MinaMesh, graphql::QueryNetworkId};
+use crate::{graphql::QueryNetworkId, MinaMesh};
 
 impl MinaMesh {
   pub async fn network_health_check(self, network_identifier: NetworkIdentifier) -> Result<bool> {

src/api/network_list.rs

@@ -2,7 +2,7 @@ use anyhow::Result;
 use cynic::QueryBuilder;
 pub use mesh::models::{NetworkIdentifier, NetworkListResponse};
 
-use crate::{MinaMesh, graphql::QueryNetworkId};
+use crate::{graphql::QueryNetworkId, MinaMesh};
 
 /// https://github.com/MinaProtocol/mina/blob/985eda49bdfabc046ef9001d3c406e688bc7ec45/src/app/rosetta/lib/network.ml#L162
 impl MinaMesh {

src/api/network_status.rs

@@ -4,8 +4,8 @@ use cynic::QueryBuilder;
 pub use mesh::models::{BlockIdentifier, NetworkStatusResponse, Peer};
 
 use crate::{
-  MinaMesh, MinaMeshError,
   graphql::{Block3, DaemonStatus3, QueryNetworkStatus},
+  MinaMesh, MinaMeshError,
 };
 
 /// https://github.com/MinaProtocol/mina/blob/985eda49bdfabc046ef9001d3c406e688bc7ec45/src/app/rosetta/lib/network.ml#L201

src/commands/fetch_genesis_block_identifier.rs

@@ -1,6 +1,6 @@
-use anyhow::{Result, bail};
+use anyhow::{bail, Result};
 use clap::Args;
-use cynic::{QueryBuilder, http::ReqwestExt};
+use cynic::{http::ReqwestExt, QueryBuilder};
 
 use crate::graphql::QueryGenesisBlockIdentifier;
 

src/commands/serve.rs

@@ -1,12 +1,12 @@
 use std::sync::Arc;
 
 use anyhow::Result;
-use axum::{Json, Router, extract::State, response::IntoResponse, routing::post, serve};
+use axum::{extract::State, response::IntoResponse, routing::post, serve, Json, Router};
 use clap::Args;
 use paste::paste;
 use tokio::net::TcpListener;
 
-use crate::{MinaMesh, MinaMeshConfig, util::Wrapper};
+use crate::{util::Wrapper, MinaMesh, MinaMeshConfig};
 
 #[derive(Debug, Args)]
 #[command(about = "Start the Mina Mesh Server.")]

src/config.rs

@@ -3,7 +3,7 @@ use clap::Args;
 use mesh::models::BlockIdentifier;
 use sqlx::PgPool;
 
-use crate::{MinaMesh, graphql::GraphQLClient};
+use crate::{graphql::GraphQLClient, MinaMesh};
 
 #[derive(Debug, Args)]
 pub struct MinaMeshConfig {