feat: add CellsCommitments softfork

chain/src/chain.rs

@@ -787,6 +787,7 @@ impl ChainService {
                                     switch,
                                     Arc::clone(&txs_verify_cache),
                                     &mmr,
+                                    &txn,
                                 );
                                 contextual_block_verifier.verify(&resolved, b)
                             };

db-schema/src/lib.rs

@@ -3,7 +3,7 @@
 /// Column families alias type
 pub type Col = &'static str;
 /// Total column number
-pub const COLUMNS: u32 = 19;
+pub const COLUMNS: u32 = 20;
 /// Column store chain index
 pub const COLUMN_INDEX: Col = "0";
 /// Column store block's header
@@ -44,6 +44,8 @@ pub const COLUMN_CHAIN_ROOT_MMR: Col = "16";
 pub const COLUMN_BLOCK_FILTER: Col = "17";
 /// Column store filter data hash for client-side filtering
 pub const COLUMN_BLOCK_FILTER_HASH: Col = "18";
+/// Column store cells root MMR data
+pub const COLUMN_CELLS_ROOT_MMR: Col = "19";
 
 /// META_TIP_HEADER_KEY tracks the latest known best block header
 pub const META_TIP_HEADER_KEY: &[u8] = b"TIP_HEADER";
@@ -52,6 +54,13 @@ pub const META_CURRENT_EPOCH_KEY: &[u8] = b"CURRENT_EPOCH";
 /// META_FILTER_DATA_KEY tracks the latest built filter data block hash
 pub const META_LATEST_BUILT_FILTER_DATA_KEY: &[u8] = b"LATEST_BUILT_FILTER_DATA";
 
+/// A key prefix records the cells root mmr size by block number
+pub const CELLS_ROOT_MMR_SIZE_KEY_PREFIX: &[u8] = &[0];
+/// A key prefix records the status of out_point in cells root mmr
+pub const CELLS_ROOT_MMR_STATUS_KEY_PREFIX: &[u8] = &[1];
+/// A key prefix records the cells root mmr element by position
+pub const CELLS_ROOT_MMR_ELEMENT_KEY_PREFIX: &[u8] = &[2];
+
 /// CHAIN_SPEC_HASH_KEY tracks the hash of chain spec which created current database
 pub const CHAIN_SPEC_HASH_KEY: &[u8] = b"chain-spec-hash";
 /// MIGRATION_VERSION_KEY tracks the current database version.

rpc/src/module/test.rs

@@ -540,12 +540,31 @@ impl IntegrationTestRpc for IntegrationTestRpcImpl {
         Ok(tx_hash.unpack())
     }
 
-    fn generate_block_with_template(&self, block_template: BlockTemplate) -> Result<H256> {
+    fn generate_block_with_template(&self, mut block_template: BlockTemplate) -> Result<H256> {
         let dao_field = self.calculate_dao_field(block_template.clone())?;
 
-        let mut update_dao_template = block_template;
-        update_dao_template.dao = dao_field;
-        let block = update_dao_template.into();
+        let cellbase = packed::Transaction::from(block_template.cellbase.data.clone()).into_view();
+        let txs = block_template
+            .transactions
+            .iter()
+            .map(|t| packed::Transaction::from(t.data.clone()).into_view())
+            .collect::<Vec<_>>();
+        let extension = self
+            .shared
+            .snapshot()
+            .build_extension(&cellbase, txs.iter())
+            .map_err(|err| {
+                error!(
+                    "build extension error when generating block \
+                 with block template, error: {:?}",
+                    err
+                );
+                RPCError::invalid_params(err.to_string())
+            })?;
+
+        block_template.dao = dao_field;
+        block_template.extension = extension.map(|e| e.into());
+        let block = block_template.into();
         self.process_and_announce_block(block)
     }
 

spec/src/lib.rs

@@ -532,7 +532,17 @@ impl ChainSpec {
                     active_mode: ActiveMode::Always,
                     threshold: TESTNET_ACTIVATION_THRESHOLD,
                 };
+                let cells_commitments = Deployment {
+                    bit: 2,
+                    start: 0,
+                    timeout: 0,
+                    min_activation_epoch: 0,
+                    period: 10,
+                    active_mode: ActiveMode::Always,
+                    threshold: TESTNET_ACTIVATION_THRESHOLD,
+                };
                 deployments.insert(DeploymentPos::LightClient, light_client);
+                deployments.insert(DeploymentPos::CellsCommitments, cells_commitments);
                 Some(deployments)
             }
         }

spec/src/versionbits/convert.rs

@@ -48,6 +48,7 @@ impl From<DeploymentPos> for JsonDeploymentPos {
         match pos {
             DeploymentPos::Testdummy => JsonDeploymentPos::Testdummy,
             DeploymentPos::LightClient => JsonDeploymentPos::LightClient,
+            DeploymentPos::CellsCommitments => JsonDeploymentPos::CellsCommitments,
         }
     }
 }

spec/src/versionbits/mod.rs

@@ -59,8 +59,10 @@ pub enum ActiveMode {
 pub enum DeploymentPos {
     /// Dummy
     Testdummy,
-    /// light client protocol
+    /// Light client protocol
     LightClient,
+    /// Tranaction output commitments
+    CellsCommitments,
 }
 
 /// VersionbitsIndexer

store/Cargo.toml

@@ -20,6 +20,7 @@ ckb-app-config = { path = "../util/app-config", version = "= 0.112.0-pre" }
 ckb-db-schema = { path = "../db-schema", version = "= 0.112.0-pre" }
 ckb-freezer = { path = "../freezer", version = "= 0.112.0-pre" }
 ckb-merkle-mountain-range =  { git = "https://github.com/quake/merkle-mountain-range", branch = "quake/updatable" }
+ckb-logger = { path = "../util/logger", version = "= 0.112.0-pre" }
 
 [dev-dependencies]
 tempfile.workspace = true

store/src/cell.rs

@@ -1,6 +1,11 @@
 use crate::{ChainStore, StoreTransaction};
-use ckb_error::Error;
-use ckb_types::{core::BlockView, packed, prelude::*};
+use ckb_error::{Error, InternalErrorKind};
+use ckb_types::{
+    core::{BlockNumber, BlockView},
+    packed,
+    prelude::*,
+    utilities::merkle_mountain_range::{hash_out_point_and_status, CellStatus},
+};
 use std::collections::HashMap;
 
 /**
@@ -26,6 +31,39 @@ use std::collections::HashMap;
 pub fn attach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<(), Error> {
     let transactions = block.transactions();
 
+    // update cells root mmr
+    let block_number = block.header().number();
+    let mut cells_root_mmr = txn.cells_root_mmr(block_number);
+    for tx in transactions.iter() {
+        for input in tx.inputs().into_iter() {
+            let out_point = input.previous_output();
+            // cellbase and genesis block's tx may not have previous output
+            if let Some(mut cell_status) = txn.get_cells_root_mmr_status(&out_point) {
+                cells_root_mmr
+                    .update(
+                        cell_status.mmr_position,
+                        hash_out_point_and_status(&out_point, cell_status.created_by, block_number),
+                    )
+                    .map_err(|e| InternalErrorKind::MMR.other(e))?;
+                cell_status.mark_as_consumed(block_number);
+                txn.insert_cells_root_mmr_status(&out_point, &cell_status)?;
+            }
+        }
+
+        for out_point in tx.output_pts().into_iter() {
+            let hash = hash_out_point_and_status(&out_point, block_number, BlockNumber::MAX);
+            let mmr_position = cells_root_mmr
+                .push(hash)
+                .map_err(|e| InternalErrorKind::MMR.other(e))?;
+            let cell_status = CellStatus::new(mmr_position, block_number);
+            txn.insert_cells_root_mmr_status(&out_point, &cell_status)?;
+        }
+    }
+    cells_root_mmr
+        .commit()
+        .map_err(|e| InternalErrorKind::MMR.other(e))?;
+    txn.insert_cells_root_mmr_size(block_number, cells_root_mmr.mmr_size())?;
+
     // add new live cells
     let new_cells = transactions
         .iter()
@@ -84,6 +122,46 @@ pub fn attach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<()
 /// Undoes the effects of this block on the live cell set.
 pub fn detach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<(), Error> {
     let transactions = block.transactions();
+
+    // undo cells root mmr updates
+    let block_number = block.header().number();
+    let mut cells_root_mmr = txn.cells_root_mmr(block_number);
+
+    for tx in transactions.iter() {
+        for input in tx.inputs().into_iter() {
+            let out_point = input.previous_output();
+            if let Some(mut cell_status) = txn.get_cells_root_mmr_status(&out_point) {
+                cells_root_mmr
+                    .update(
+                        cell_status.mmr_position,
+                        hash_out_point_and_status(
+                            &out_point,
+                            cell_status.created_by,
+                            BlockNumber::MAX,
+                        ),
+                    )
+                    .map_err(|e| InternalErrorKind::MMR.other(e))?;
+                cell_status.mark_as_live();
+                txn.insert_cells_root_mmr_status(&out_point, &cell_status)?;
+            }
+        }
+
+        for out_point in tx.output_pts().into_iter() {
+            txn.delete_cells_root_mmr_status(&out_point)?;
+        }
+    }
+    cells_root_mmr
+        .commit()
+        .map_err(|e| InternalErrorKind::MMR.other(e))?;
+
+    let current_mmr_size = txn.get_cells_root_mmr_size(block_number);
+    let pre_mmr_size = txn.get_cells_root_mmr_size(block_number - 1);
+    for pos in pre_mmr_size..current_mmr_size {
+        txn.delete_cells_root_mmr_element(pos, block_number)?;
+    }
+    txn.delete_cells_root_mmr_size(block_number)?;
+
+    // restore inputs
     let mut input_pts = HashMap::with_capacity(transactions.len());
 
     for tx in transactions.iter().skip(1) {
@@ -95,7 +173,6 @@ pub fn detach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<()
         }
     }
 
-    // restore inputs
     // skip cellbase
     let undo_deads = input_pts
         .iter()

store/src/store.rs

@@ -5,11 +5,13 @@ use ckb_db::{
     DBPinnableSlice,
 };
 use ckb_db_schema::{
-    Col, COLUMN_BLOCK_BODY, COLUMN_BLOCK_EPOCH, COLUMN_BLOCK_EXT, COLUMN_BLOCK_EXTENSION,
-    COLUMN_BLOCK_FILTER, COLUMN_BLOCK_FILTER_HASH, COLUMN_BLOCK_HEADER, COLUMN_BLOCK_PROPOSAL_IDS,
-    COLUMN_BLOCK_UNCLE, COLUMN_CELL, COLUMN_CELL_DATA, COLUMN_CELL_DATA_HASH,
-    COLUMN_CHAIN_ROOT_MMR, COLUMN_EPOCH, COLUMN_INDEX, COLUMN_META, COLUMN_TRANSACTION_INFO,
-    COLUMN_UNCLES, META_CURRENT_EPOCH_KEY, META_LATEST_BUILT_FILTER_DATA_KEY, META_TIP_HEADER_KEY,
+    Col, CELLS_ROOT_MMR_ELEMENT_KEY_PREFIX, CELLS_ROOT_MMR_SIZE_KEY_PREFIX,
+    CELLS_ROOT_MMR_STATUS_KEY_PREFIX, COLUMN_BLOCK_BODY, COLUMN_BLOCK_EPOCH, COLUMN_BLOCK_EXT,
+    COLUMN_BLOCK_EXTENSION, COLUMN_BLOCK_FILTER, COLUMN_BLOCK_FILTER_HASH, COLUMN_BLOCK_HEADER,
+    COLUMN_BLOCK_PROPOSAL_IDS, COLUMN_BLOCK_UNCLE, COLUMN_CELL, COLUMN_CELLS_ROOT_MMR,
+    COLUMN_CELL_DATA, COLUMN_CELL_DATA_HASH, COLUMN_CHAIN_ROOT_MMR, COLUMN_EPOCH, COLUMN_INDEX,
+    COLUMN_META, COLUMN_TRANSACTION_INFO, COLUMN_UNCLES, META_CURRENT_EPOCH_KEY,
+    META_LATEST_BUILT_FILTER_DATA_KEY, META_TIP_HEADER_KEY,
 };
 use ckb_freezer::Freezer;
 use ckb_types::{
@@ -20,6 +22,8 @@ use ckb_types::{
     },
     packed::{self, OutPoint},
     prelude::*,
+    utilities::merkle_mountain_range::CellStatus,
+    H256,
 };
 
 /// The `ChainStore` trait provides chain data store interface
@@ -576,6 +580,44 @@ pub trait ChainStore: Send + Sync + Sized {
             })
     }
 
+    /// Gets cells root mmr size by block number
+    fn get_cells_root_mmr_size(&self, block_number: BlockNumber) -> u64 {
+        let start_key = [CELLS_ROOT_MMR_SIZE_KEY_PREFIX, &block_number.to_be_bytes()].concat();
+        self.get_iter(
+            COLUMN_CELLS_ROOT_MMR,
+            IteratorMode::From(&start_key, Direction::Reverse),
+        )
+        .take_while(|(key, _)| key.starts_with(CELLS_ROOT_MMR_SIZE_KEY_PREFIX))
+        .next()
+        .map(|(_key, value)| u64::from_le_bytes(value.as_ref().try_into().expect("stored u64")))
+        .unwrap_or_default()
+    }
+
+    /// Gets cells root mmr status by out point
+    fn get_cells_root_mmr_status(&self, out_point: &OutPoint) -> Option<CellStatus> {
+        let key = [CELLS_ROOT_MMR_STATUS_KEY_PREFIX, out_point.as_slice()].concat();
+        self.get(COLUMN_CELLS_ROOT_MMR, &key)
+            .map(|slice| CellStatus::new_unchecked(slice.as_ref()))
+    }
+
+    /// Gets cells root mmr element by position and block number
+    fn get_cells_root_mmr_element(&self, position: u64, block_number: BlockNumber) -> Option<H256> {
+        // block_number is stored as big endian for prefix search in lexicographical order
+        let start_key = [
+            CELLS_ROOT_MMR_ELEMENT_KEY_PREFIX,
+            &position.to_le_bytes(),
+            &block_number.to_be_bytes(),
+        ]
+        .concat();
+        self.get_iter(
+            COLUMN_CELLS_ROOT_MMR,
+            IteratorMode::From(&start_key, Direction::Reverse),
+        )
+        .take_while(|(key, _)| key.starts_with(&start_key[0..9]))
+        .next()
+        .map(|(_key, value)| H256::from_slice(value.as_ref()).expect("stored H256"))
+    }
+
     /// Gets ancestor block header by a base block hash and number
     fn get_ancestor(&self, base: &packed::Byte32, number: BlockNumber) -> Option<HeaderView> {
         let header = self.get_block_header(base)?;