diff --git a/.db-versions.yml b/.db-versions.yml new file mode 100644 index 000000000..db6802319 --- /dev/null +++ b/.db-versions.yml @@ -0,0 +1,4 @@ +current_version: 0 +versions: + - version: 0 + pr: 372 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a5db7b9a7..5b3c07739 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ -# Pull Request type +## Pull Request type @@ -33,6 +33,11 @@ Resolves: #NA + ## Other information diff --git a/.github/labels.yml b/.github/labels.yml index 897620279..20c2550b2 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -2,6 +2,9 @@ - name: "breaking-change" color: ee0701 description: "A change that changes the API or breaks backward compatibility for users." +- name: "bump_db" + color: ee0701 + description: "Changes requiring a database version increment." - name: "bugfix" color: ee0701 description: diff --git a/.github/workflows/db-version.yml b/.github/workflows/db-version.yml new file mode 100644 index 000000000..b64c83fb8 --- /dev/null +++ b/.github/workflows/db-version.yml @@ -0,0 +1,46 @@ +--- +name: DB Version Management + +on: + workflow_dispatch: + workflow_call: + +jobs: + update-db-version: + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'bump_db') + steps: + - uses: actions/checkout@v3 + + - name: Install yq + run: sudo apt-get install -y yq + + - name: Check if PR already bumped + id: check_bump + run: | + PR_NUM="${{ github.event.pull_request.number }}" + if yq -e ".versions[] | select(.pr == ${PR_NUM})" .db-versions.yml > /dev/null 2>&1; then + echo "already_bumped=true" >> $GITHUB_OUTPUT + else + echo "already_bumped=false" >> $GITHUB_OUTPUT + fi + + - name: Configure Git + if: steps.check_bump.outputs.already_bumped == 'false' + run: | + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Update DB Version + if: steps.check_bump.outputs.already_bumped == 'false' + run: | + ./scripts/update-db-version.sh "${{ github.event.pull_request.number }}" + + - name: Commit and Push + if: steps.check_bump.outputs.already_bumped == 'false' + run: | + if [[ -n "$(git status --porcelain)" ]]; then + git add .db-versions.toml + git commit -m "chore: bump db version" + git push origin HEAD + fi diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 1a076f13b..915d41ad1 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -14,9 +14,15 @@ permissions: pull-requests: write jobs: + update_db_version: + name: Update DB Version + if: github.event.pull_request.draft == false + uses: ./.github/workflows/db-version.yml + linters: name: Run linters - if: github.event.pull_request.draft == false + needs: update_db_version + if: ${{ always() }} uses: ./.github/workflows/linters.yml rust_check: @@ -26,21 +32,24 @@ jobs: linters_cargo: name: Run Cargo linters - if: github.event.pull_request.draft == false - uses: ./.github/workflows/linters-cargo.yml needs: rust_check + uses: ./.github/workflows/linters-cargo.yml coverage: name: Run Coverage - if: github.event.pull_request.draft == false - uses: ./.github/workflows/coverage.yml + needs: update_db_version + if: ${{ always() }} secrets: inherit + uses: ./.github/workflows/coverage.yml build: name: Build Madara + needs: update_db_version + if: ${{ always() }} uses: ./.github/workflows/build.yml js_test: name: Run JS Tests - uses: ./.github/workflows/starknet-js-test.yml needs: build + if: ${{ always() }} + uses: ./.github/workflows/starknet-js-test.yml diff --git a/README.md b/README.md index c3b71ad31..47906d2ff 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Madara is a powerful Starknet client written in Rust. - [Madara-specific JSON-RPC Methods](#madara-specific-json-rpc-methods) - [Example of Calling a JSON-RPC Method](#example-of-calling-a-json-rpc-method) - 📚 [Database Migration](#-database-migration) + - [Database Version Management](#database-version-management) - [Warp Update](#warp-update) - [Running without `--warp-update-sender`](#running-without---warp-update-sender) - ✅ [Supported Features](#-supported-features) @@ -545,10 +546,36 @@ which is returned with each websocket response. [⬅️ back to top](#-madara-starknet-client) -When migration to a newer version of Madara you might need to update your -database. Instead of re-synchronizing the entirety of your chain's state from -genesis, you can use Madara's **warp update** feature. This is essentially a -form of trusted sync with better performances as it is run from a local source. +### Database Version Management + +The database version management system ensures compatibility between Madara's +binary and database versions. +When you encounter a version mismatch error, it means your database schema needs +to be updated to match your current binary version. + +When you see: + +```console +Error: Database version 41 is not compatible with current binary. Expected version 42 +``` + +This error indicates that: + +1. Your current binary requires database version 42 +2. Your database is still at version 41 +3. Migration is required before you can continue + +> [!IMPORTANT] +> Don't panic! Your data is safe, but you need to migrate it before continuing. + +To migrate your database, you have two options: + +1. Use Madara's **warp update** feature (recommended) +2. Re-synchronize from genesis (not recommended) + +The warp update feature provides a trusted sync from a local source, offering +better performance than re-synchronizing the entirety of your chain's state +from genesis. ### Warp Update diff --git a/crates/madara/client/db/build.rs b/crates/madara/client/db/build.rs new file mode 100644 index 000000000..899d25809 --- /dev/null +++ b/crates/madara/client/db/build.rs @@ -0,0 +1,145 @@ +//! Database version management for build-time validation +//! +//! This build script: +//! 1. Reads the current database version from `.db-versions.yml` in project root +//! 2. Injects it as `DB_VERSION` environment variable for runtime checks +//! 3. Ensures version file is well-formatted +//! +//! # File format +//! The version file must be a YAML file with the following structure: +//! ```yaml +//! current_version: 42 +//! versions: +//! - version: 42 +//! pr: 123 +//! - version: 41 +//! pr: 120 +//! ``` +//! +//! # Environment variables +//! - `CARGO_MANIFEST_DIR`: Set by cargo, path to the current crate +//! +//! # Outputs +//! - `cargo:rustc-env=DB_VERSION=X`: Current database version +//! - `cargo:rerun-if-changed=.db-versions.yml`: Rebuild if version changes +//! +//! # Errors +//! Fails the build if: +//! - Version file is missing or malformed +//! - Version number cannot be parsed +//! - Cannot find project root directory + +use std::borrow::Cow; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; + +const DB_VERSION_FILE: &str = ".db-versions.yml"; +const PARENT_LEVELS: usize = 4; + +#[allow(clippy::print_stderr)] +fn main() { + if let Err(e) = get_db_version() { + eprintln!("Failed to get DB version: {}", e); + std::process::exit(1); + } +} + +#[derive(Debug)] +enum BuildError { + EnvVar(env::VarError), + Io(std::io::Error), + Parse(Cow<'static, str>), +} + +impl std::fmt::Display for BuildError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BuildError::EnvVar(e) => write!(f, "Environment variable error: {}", e), + BuildError::Io(e) => write!(f, "IO error: {}", e), + BuildError::Parse(msg) => write!(f, "Parse error: {}", msg), + } + } +} + +impl From for BuildError { + fn from(e: env::VarError) -> Self { + BuildError::EnvVar(e) + } +} + +impl From for BuildError { + fn from(e: std::io::Error) -> Self { + BuildError::Io(e) + } +} + +fn get_db_version() -> Result<(), BuildError> { + let manifest_dir = env::var("CARGO_MANIFEST_DIR")?; + let root_dir = get_parents(&PathBuf::from(manifest_dir), PARENT_LEVELS)?; + let file_path = root_dir.join(DB_VERSION_FILE); + + let content = fs::read_to_string(&file_path).map_err(|e| { + BuildError::Io(std::io::Error::new(e.kind(), format!("Failed to read {}: {}", file_path.display(), e))) + })?; + + let current_version = parse_version(&content)?; + + println!("cargo:rerun-if-changed={}", DB_VERSION_FILE); + println!("cargo:rustc-env=DB_VERSION={}", current_version); + + Ok(()) +} + +fn parse_version(content: &str) -> Result { + content + .lines() + .find(|line| line.starts_with("current_version:")) + .ok_or_else(|| BuildError::Parse(Cow::Borrowed("Could not find current_version")))? + .split(':') + .nth(1) + .ok_or_else(|| BuildError::Parse(Cow::Borrowed("Invalid current_version format")))? + .trim() + .parse() + .map_err(|_| BuildError::Parse(Cow::Borrowed("Could not parse current_version as u32"))) +} + +fn get_parents(path: &Path, n: usize) -> Result { + let mut path = path.to_path_buf(); + for _ in 0..n { + path = path + .parent() + .ok_or(BuildError::Io(std::io::Error::new(std::io::ErrorKind::NotFound, "Parent not found")))? + .to_path_buf(); + } + Ok(path) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + #[test] + fn test_parse_version_valid() { + let content = "current_version: 42\nother: stuff"; + assert_eq!(parse_version(content).unwrap(), 42); + } + + #[test] + fn test_parse_version_invalid_format() { + let content = "wrong_format"; + assert!(matches!(parse_version(content), Err(BuildError::Parse(_)))); + } + + #[test] + fn test_get_parents() { + let temp = TempDir::new().unwrap(); + let path = temp.path().join("a").join("b").join("c"); + fs::create_dir_all(&path).unwrap(); + + let result = get_parents(&path, 2).unwrap(); + assert_eq!(result, temp.path().join("a")); + } +} diff --git a/crates/madara/client/db/src/db_version.rs b/crates/madara/client/db/src/db_version.rs new file mode 100644 index 000000000..2b25f84d9 --- /dev/null +++ b/crates/madara/client/db/src/db_version.rs @@ -0,0 +1,165 @@ +//! Database version compatibility checker +//! +//! This module ensures database version compatibility with the current binary. +//! The version check prevents data corruption from version mismatches between +//! database files and binary versions. +//! +//! # Version File +//! The version is stored in a `.db-version` file in the database directory. +//! This file contains a single number representing the database version. +//! + +use std::fs; +use std::path::Path; + +/// Database version from build-time, injected by build.rs +const REQUIRED_DB_VERSION: &str = env!("DB_VERSION"); + +/// File name for database version +/// This file contains a single number representing the database version +const DB_VERSION_FILE: &str = ".db-version"; + +/// Errors that can occur during version checking +#[derive(Debug, thiserror::Error)] +pub enum DbVersionError { + /// The database version doesn't match the binary version + #[error( + "Database version {db_version} is not compatible with current binary. Expected version {required_version}" + )] + IncompatibleVersion { + /// Version found in database + db_version: u32, + /// Version required by binary + required_version: u32, + }, + + /// Error reading or writing the version file + #[error("Failed to read database version: {0}")] + VersionReadError(String), +} + +/// Checks database version compatibility with current binary. +/// +/// # Arguments +/// * `path` - Path to the database directory +/// +/// # Returns +/// * `Ok(None)` - New database created with current version +/// * `Ok(Some(version))` - Existing database with compatible version +/// * `Err(DbVersionError)` - Version mismatch or IO error +/// +/// # Examples +/// ```ignore +/// use std::path::Path; +/// use crate::db_version::check_db_version; +/// +/// let db_path = Path::new("test_db"); +/// match check_db_version(db_path) { +/// Ok(None) => println!("Created new database"), +/// Ok(Some(v)) => println!("Database version {} is compatible", v), +/// Err(e) => eprintln!("Error: {}", e), +/// } +/// ``` +/// +pub fn check_db_version(path: &Path) -> Result, DbVersionError> { + let required_db_version = + REQUIRED_DB_VERSION.parse::().expect("REQUIRED_DB_VERSION is checked at compile time"); + + // Create directory if it doesn't exist + if !path.exists() { + fs::create_dir_all(path).map_err(|e| DbVersionError::VersionReadError(e.to_string()))?; + } + + let file_path = path.join(DB_VERSION_FILE); + if !file_path.exists() { + // Initialize new database with current version + fs::write(&file_path, REQUIRED_DB_VERSION).map_err(|e| DbVersionError::VersionReadError(e.to_string()))?; + Ok(None) + } else { + // Check existing database version + let version = fs::read_to_string(&file_path).map_err(|e| DbVersionError::VersionReadError(e.to_string()))?; + let version = version.trim().parse::().map_err(|_| DbVersionError::VersionReadError(version))?; + + if version != required_db_version { + return Err(DbVersionError::IncompatibleVersion { + db_version: version, + required_version: required_db_version, + }); + } + Ok(Some(version)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + /// Helper function to create a test database directory + fn setup_test_db() -> TempDir { + TempDir::new().unwrap() + } + + #[test] + fn test_new_database() { + let temp_dir = setup_test_db(); + let result = check_db_version(temp_dir.path()).unwrap(); + assert!(result.is_none()); + + // Verify version file was created + let version_file = temp_dir.path().join(DB_VERSION_FILE); + assert!(version_file.exists()); + + // Verify content + let content = fs::read_to_string(version_file).unwrap(); + assert_eq!(content, REQUIRED_DB_VERSION); + } + + #[test] + fn test_compatible_version() { + let temp_dir = setup_test_db(); + let version_file = temp_dir.path().join(DB_VERSION_FILE); + + // Create version file with current version + fs::write(&version_file, REQUIRED_DB_VERSION).unwrap(); + + let result = check_db_version(temp_dir.path()).unwrap(); + assert_eq!(result, Some(REQUIRED_DB_VERSION.parse().unwrap())); + } + + #[test] + fn test_incompatible_version() { + let temp_dir = setup_test_db(); + let version_file = temp_dir.path().join(DB_VERSION_FILE); + + // Create version file with different version + let incompatible_version = REQUIRED_DB_VERSION.parse::().unwrap().checked_add(1).unwrap().to_string(); + fs::write(version_file, incompatible_version).unwrap(); + + let err = check_db_version(temp_dir.path()).unwrap_err(); + assert!(matches!(err, DbVersionError::IncompatibleVersion { .. })); + } + + #[test] + fn test_invalid_version_format() { + let temp_dir = setup_test_db(); + let version_file = temp_dir.path().join(DB_VERSION_FILE); + + // Create version file with invalid content + fs::write(version_file, "invalid").unwrap(); + + let err = check_db_version(temp_dir.path()).unwrap_err(); + assert!(matches!(err, DbVersionError::VersionReadError(..))); + } + + #[test] + fn test_creates_missing_directory() { + let temp_dir = setup_test_db(); + let db_path = temp_dir.path().join(DB_VERSION_FILE); + + let result = check_db_version(&db_path).unwrap(); + assert!(result.is_none()); + assert!(db_path.exists()); + assert!(db_path.join(".db-version").exists()); + } +} diff --git a/crates/madara/client/db/src/lib.rs b/crates/madara/client/db/src/lib.rs index 6811cf3f6..763531649 100644 --- a/crates/madara/client/db/src/lib.rs +++ b/crates/madara/client/db/src/lib.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use std::{fmt, fs}; use tokio::sync::{mpsc, oneshot}; +mod db_version; mod error; mod rocksdb_options; mod rocksdb_snapshot; @@ -398,6 +399,12 @@ impl MadaraBackend { chain_config: Arc, trie_log_config: TrieLogConfig, ) -> anyhow::Result> { + // check if the db version is compatible with the current binary + tracing::debug!("checking db version"); + if let Some(db_version) = db_version::check_db_version(&db_config_dir).context("Checking database version")? { + tracing::debug!("version of existing db is {db_version}"); + } + let db_path = db_config_dir.join("db"); // when backups are enabled, a thread is spawned that owns the rocksdb BackupEngine (it is not thread safe) and it receives backup requests using a mpsc channel diff --git a/flake.nix b/flake.nix index c41ac5eb2..6fbd4c5cd 100644 --- a/flake.nix +++ b/flake.nix @@ -23,6 +23,7 @@ protobuf nodePackages.prettier taplo-cli + yq ]; buildInputs = with pkgs; [ diff --git a/scripts/update-db-version.sh b/scripts/update-db-version.sh new file mode 100755 index 000000000..4d01023b7 --- /dev/null +++ b/scripts/update-db-version.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +# +# Database version management script +# +# This script updates the database version tracking file when schema changes occur. +# It's typically called by CI when a PR with the 'bump_db' label is merged. +# +# Requirements: yq (https://github.com/mikefarah/yq/) +# +# Usage: +# ./update-db-version.sh PR_NUMBER +# +# Arguments: +# PR_NUMBER - The pull request number that introduced the schema changes +# +# File format (.db-versions.yml): +# current_version: 41 +# versions: +# - version: 41 +# pr: 120 +# +# Environment: +# No specific environment variables required +# +# Exit codes: +# 0 - Success +# 1 - Usage error +# 1 - Missing dependencies +# 1 - File read/write error +# 1 - Version parsing error +# 1 - PR already exists +# +# Example: +# ./update-db-version.sh 123 +# Successfully updated DB version from 41 to 42 (PR #123) + +set -euo pipefail + +FILE=".db-versions.yml" + +[ $# -eq 1 ] || { echo "Usage: $0 PR_NUMBER" >&2; exit 1; } +PR_NUMBER="$1" + +command -v yq >/dev/null 2>&1 || { echo "Error: yq is required but not installed" >&2; exit 1; } + +[ -f "$FILE" ] || { echo "Error: $FILE not found" >&2; exit 1; } + +# Check duplicate PR +yq -e ".versions[] | select(.pr == $PR_NUMBER)" "$FILE" >/dev/null 2>&1 && { + echo "Error: PR #${PR_NUMBER} already exists in version history" >&2 + exit 1 +} + + +# Get and validate current version +CURRENT_VERSION=$(yq '.current_version' "$FILE") +[[ "$CURRENT_VERSION" =~ ^[0-9]+$ ]] || { + echo "Error: Invalid current_version in $FILE" >&2 + exit 1 +} + +# Increment the version +NEW_VERSION=$((CURRENT_VERSION + 1)) + +# Update version and append to history +yq -i -y ".current_version = $NEW_VERSION | + .versions = [{\"version\": $NEW_VERSION, \"pr\": $PR_NUMBER}] + .versions" "$FILE" + +echo "Successfully updated DB version to ${NEW_VERSION} (PR #${PR_NUMBER})" \ No newline at end of file