Skip to content

Commit

Permalink
update db exporter resource
Browse files Browse the repository at this point in the history
  • Loading branch information
@nkysg
nkysg committed Feb 17, 2024
1 parent e662a7f commit 3887a87
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 108 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions cmd/db-exporter/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ starcoin-vm-runtime = { workspace = true }
futures = { workspace = true }
rayon = { workspace = true }
num_cpus = { workspace = true }
starcoin-rpc-api = { workspace = true }

[package]
authors = { workspace = true }
Expand Down
180 changes: 83 additions & 97 deletions cmd/db-exporter/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-License-Identifier: Apache-2.0

use anyhow::{bail, format_err, Result};
use bcs_ext::{BCSCodec, Sample};
use bcs_ext::Sample;
use clap::{IntoApp, Parser};
use csv::Writer;
use db_exporter::{
Expand All @@ -22,6 +22,8 @@ use starcoin_consensus::Consensus;
use starcoin_crypto::HashValue;
use starcoin_genesis::Genesis;
use starcoin_resource_viewer::{AnnotatedMoveStruct, AnnotatedMoveValue, MoveValueAnnotator};
use starcoin_rpc_api::types::StrView;
use starcoin_state_tree::StateTree;
use starcoin_statedb::{ChainStateDB, ChainStateReader, ChainStateWriter};
use starcoin_storage::{
block::FailedBlock,
Expand All @@ -36,6 +38,7 @@ use starcoin_storage::{
use starcoin_transaction_builder::{
build_signed_empty_txn, create_signed_txn_with_association_account, DEFAULT_MAX_GAS_AMOUNT,
};
use starcoin_types::account_state::AccountState;
use starcoin_types::{
account::{peer_to_peer_txn, Account, DEFAULT_EXPIRATION_TIME},
account_address::AccountAddress,
Expand All @@ -45,20 +48,20 @@ use starcoin_types::{
transaction::Transaction,
};
use starcoin_vm_runtime::starcoin_vm::StarcoinVM;
use starcoin_vm_types::access_path::DataType;
use starcoin_vm_types::{
account_config::stc_type_tag,
genesis_config::ConsensusStrategy,
identifier::Identifier,
language_storage::{ModuleId, StructTag},
parser::parse_struct_tag,
transaction::{ScriptFunction, SignedUserTransaction, TransactionPayload},
};
use std::{
collections::HashMap,
fmt::{Debug, Formatter},
fs::{File, OpenOptions},
io::{BufRead, BufReader, Write},
path::{Path, PathBuf},
path::PathBuf,
str::FromStr,
sync::{
atomic::{AtomicU64, Ordering},
Expand Down Expand Up @@ -392,29 +395,24 @@ pub struct ApplySnapshotOptions {
#[derive(Debug, Clone, Parser)]
#[clap(name = "export-resource", about = "onchain resource exporter")]
pub struct ExportResourceOptions {
#[clap(long, short = 'n')]
/// Chain Network, like main, proxima
pub net: BuiltinNetworkID,
#[clap(long, short = 'o', parse(from_os_str))]
/// output file, like accounts.csv
pub output: PathBuf,
#[clap(long, short = 'i', parse(from_os_str))]
/// starcoin node db path. like ~/.starcoin/barnard/starcoindb/db/starcoindb
/// starcoin node db path. like ~/.starcoin/barnard
pub db_path: PathBuf,

#[clap(long)]
/// block hash of the snapshot.
pub block_hash: HashValue,
#[clap(long, short = 'b')]
pub block_number: Option<BlockNumber>,

#[clap(
short='r',
default_value = "0x1::Account::Balance<0x1::STC::STC>",
parse(try_from_str=parse_struct_tag)
help = "resource struct tag,",
default_value = "0x1::Account::Balance<0x1::STC::STC>"
)]
/// resource struct tag.
resource_type: StructTag,

#[clap(min_values = 1, required = true)]
/// fields of the struct to output. it use pointer syntax of serde_json.
/// like: /authentication_key /sequence_number /deposit_events/counter /token/value
pub fields: Vec<String>,
resource_type: StrView<StructTag>,
}

#[derive(Debug, Parser)]
Expand Down Expand Up @@ -574,16 +572,12 @@ async fn main() -> anyhow::Result<()> {
Cmd::ExportResource(option) => {
#[cfg(target_os = "linux")]
let guard = pprof::ProfilerGuard::new(100).unwrap();
let output = option.output.as_path();
let block_hash = option.block_hash;
let resource = option.resource_type.clone();
// let result = apply_block(option.to_path, option.input_path, option.net, verifier);
export_resource(
option.db_path.display().to_string().as_str(),
output,
block_hash,
resource,
option.fields.as_slice(),
option.db_path,
option.output,
option.net,
option.block_number,
option.resource_type.0.clone(),
)?;
#[cfg(target_os = "linux")]
if let Ok(report) = guard.report().build() {
Expand Down Expand Up @@ -1818,100 +1812,92 @@ pub struct AccountData<R: Serialize> {
}

pub fn export_resource(
db: &str,
output: &Path,
block_hash: HashValue,
from_dir: PathBuf,
output: PathBuf,
network: BuiltinNetworkID,
block_number: Option<BlockNumber>,
resource_struct_tag: StructTag,
fields: &[String],
) -> anyhow::Result<()> {
let net = ChainNetwork::new_builtin(network);
let db_storage = DBStorage::open_with_cfs(
db,
from_dir.join("starcoindb/db/starcoindb"),
StorageVersion::current_version()
.get_column_family_names()
.to_vec(),
true,
Default::default(),
None,
)?;
let storage = Storage::new(StorageInstance::new_db_instance(db_storage))?;
let storage = Arc::new(storage);
let block = storage
.get_block(block_hash)?
.ok_or_else(|| anyhow::anyhow!("block {} not exist", block_hash))?;
let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
db_storage,
))?);
let (chain_info, _) =
Genesis::init_and_check_storage(&net, storage.clone(), from_dir.as_ref())?;
let chain = BlockChain::new(
net.time_service(),
chain_info.head().id(),
storage.clone(),
None,
)
.expect("create block chain should success.");
let cur_num = block_number.unwrap_or(chain_info.head().number());
let block = chain
.get_block_by_number(cur_num)?
.ok_or_else(|| format_err!("get block by number {} error", cur_num))?;

let root = block.header.state_root();
let statedb = ChainStateDB::new(storage, Some(root));
let statedb = ChainStateDB::new(storage.clone(), Some(root));
let value_annotator = MoveValueAnnotator::new(&statedb);

let mut csv_writer = csv::WriterBuilder::new().from_path(output)?;
let state_tree = StateTree::<AccountAddress>::new(storage.clone(), Some(root));

// write csv header.
{
csv_writer.write_field("address")?;
for f in fields {
csv_writer.write_field(f)?;
}
csv_writer.write_record(None::<&[u8]>)?;
}

use std::time::Instant;
let mut file = File::create(output)?;

let now = Instant::now();

let global_states_iter = statedb.dump_iter()?;
println!("t1: {}", now.elapsed().as_millis());
let global_states = state_tree.dump()?;

use std::time::Instant;
let now = Instant::now();
let mut t1_sum = 0;
let mut t2_sum = 0;
let mut loop_count = 0;
let mut now3 = Instant::now();
let mut iter_time = 0;
for (account_address, account_state_set) in global_states_iter {
iter_time += now3.elapsed().as_nanos();

loop_count += 1;

let now1 = Instant::now();
let resource_set = account_state_set.resource_set().unwrap();
// t1_sum += now1.elapsed().as_micros();
t1_sum += now1.elapsed().as_nanos();
// println!("t1 sum in loop: {}", t1_sum);

let now2 = Instant::now();
for (k, v) in resource_set.iter() {
let struct_tag = StructTag::decode(k.as_slice())?;
if struct_tag == resource_struct_tag {
let annotated_struct =
value_annotator.view_struct(resource_struct_tag.clone(), v.as_slice())?;
let resource_struct = annotated_struct;
let resource_json_value = serde_json::to_value(MoveStruct(resource_struct))?;
let resource = Some(resource_json_value);
let record: Option<Vec<_>> = resource
.as_ref()
.map(|v| fields.iter().map(|f| v.pointer(f.as_str())).collect());
if let Some(mut record) = record {
let account_value = serde_json::to_value(account_address).unwrap();
record.insert(0, Some(&account_value));
csv_writer.serialize(record)?;
let mut sum: u128 = 0;
for (address_bytes, account_state_bytes) in global_states.iter() {
let account: AccountAddress = bcs_ext::from_bytes(address_bytes)?;
let account_state: AccountState = account_state_bytes.as_slice().try_into()?;
let resource_root = account_state.storage_roots()[DataType::RESOURCE.storage_index()];
let resource = match resource_root {
None => None,
Some(root) => {
let account_tree = StateTree::<StructTag>::new(storage.clone(), Some(root));
let data = account_tree.get(&resource_struct_tag)?;

if let Some(d) = data {
let annotated_struct =
value_annotator.view_struct(resource_struct_tag.clone(), d.as_slice())?;
let resource = annotated_struct;
let resource_json_value = serde_json::to_value(MoveStruct(resource))?;
Some(resource_json_value)
} else {
None
}
break;
}
};
if let Some(res) = resource {
let balance = (res
.get("token")
.unwrap()
.get("value")
.unwrap()
.as_f64()
.unwrap()
/ 1000000000.0) as u128;
if balance > 0 {
writeln!(file, "{} {}", account, balance)?;
sum += balance;
}
}
// let d = now2.elapsed().as_micros();
let d = now2.elapsed().as_nanos();
// println!("d is: {}", d);
t2_sum += d;
// println!("t2 sum in loop: {}", t2_sum);
now3 = Instant::now();
}
println!("iter time: {}, {}", iter_time, iter_time / 1_000_000);
println!("loop count: {}", loop_count);
println!("t1_sum: {}, {}", t1_sum, t1_sum / 1000000);
println!("t2_sum: {}, {}", t2_sum, t2_sum / 1000000);

println!("t2: {}", now.elapsed().as_millis());
csv_writer.flush()?;
writeln!(file, "total {}", sum)?;
file.flush()?;
Ok(())
}

Expand Down
11 changes: 0 additions & 11 deletions cmd/resource-exporter/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,17 +160,6 @@ impl serde::Serialize for MoveValue {
}
}

#[derive(Parser)]
struct Opt {
#[clap(subcommand)]
cmd: Option<Cmd>,
}

#[derive(Parser)]
enum Cmd {
Exporter(ExporterOptions),
}

#[derive(Debug, Clone, Parser)]
#[clap(name = "resource-exporter", about = "onchain resource exporter")]
pub struct ExporterOptions {
Expand Down

0 comments on commit 3887a87

Please sign in to comment.