diff --git a/libraries/chain/database.cpp b/libraries/chain/database.cpp index cb999b7fb0..37dd2702c3 100644 --- a/libraries/chain/database.cpp +++ b/libraries/chain/database.cpp @@ -2467,6 +2467,11 @@ namespace golos { namespace chain { modify(get_account(comment.author), [&](account_object &a) { a.posting_rewards += author_tokens; }); + + auto author_golos = asset(author_tokens, STEEM_SYMBOL); + auto benefactor_golos = asset(total_beneficiary, STEEM_SYMBOL); + auto curator_golos = asset(reward_tokens.to_uint64() - (author_tokens + total_beneficiary), STEEM_SYMBOL); + push_virtual_operation(total_comment_reward_operation(comment.author, to_string(comment.permlink), author_golos, benefactor_golos, curator_golos, comment.net_rshares.value)); } fc::uint128_t old_rshares2 = calculate_vshares(comment.net_rshares.value); diff --git a/libraries/protocol/include/golos/protocol/operations.hpp b/libraries/protocol/include/golos/protocol/operations.hpp index cb00d3e693..9adc311df3 100644 --- a/libraries/protocol/include/golos/protocol/operations.hpp +++ b/libraries/protocol/include/golos/protocol/operations.hpp @@ -87,7 +87,8 @@ namespace golos { namespace protocol { return_vesting_delegation_operation, producer_reward_operation, delegation_reward_operation, - auction_window_reward_operation + auction_window_reward_operation, + total_comment_reward_operation > operation; /*void operation_get_required_authorities( const operation& op, diff --git a/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp b/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp index 68901a2aaf..d4fc8f20c9 100644 --- a/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp +++ b/libraries/protocol/include/golos/protocol/steem_virtual_operations.hpp @@ -232,6 +232,22 @@ namespace golos { namespace protocol { account_name_type account; asset vesting_shares; }; + + struct total_comment_reward_operation : public virtual_operation { + total_comment_reward_operation() { + } + + total_comment_reward_operation(const account_name_type& a, const string &p, const asset& ar, const asset& br, const asset& cr, int64_t nr) + : author(a), permlink(p), author_reward(ar), benefactor_reward(br), curator_reward(cr), net_rshares(nr) { + } + + account_name_type author; + string permlink; + asset author_reward; + asset benefactor_reward; + asset curator_reward; + int64_t net_rshares; + }; } } //golos::protocol FC_REFLECT((golos::protocol::author_reward_operation), (author)(permlink)(sbd_payout)(steem_payout)(vesting_payout)) @@ -251,3 +267,4 @@ FC_REFLECT((golos::protocol::comment_benefactor_reward_operation), (benefactor)( FC_REFLECT((golos::protocol::return_vesting_delegation_operation), (account)(vesting_shares)) FC_REFLECT((golos::protocol::producer_reward_operation), (producer)(vesting_shares)) FC_REFLECT((golos::protocol::delegation_reward_operation), (delegator)(delegatee)(payout_strategy)(vesting_shares)) +FC_REFLECT((golos::protocol::total_comment_reward_operation), (author)(permlink)(author_reward)(benefactor_reward)(curator_reward)(net_rshares)) diff --git a/plugins/account_history/plugin.cpp b/plugins/account_history/plugin.cpp index b72d83bf9f..681775e39e 100644 --- a/plugins/account_history/plugin.cpp +++ b/plugins/account_history/plugin.cpp @@ -486,6 +486,10 @@ if (options.count(name)) { \ insert_receiver(op.account); } + void operator()(const total_comment_reward_operation& op) { + insert_dual(op.author); + } + // todo: proposal tx signers are receivers void operator()(const proposal_create_operation& op) { insert_dual(op.author); diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp index 5ebabdf0d2..18723efa6a 100644 --- a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_operations.hpp @@ -86,6 +86,7 @@ namespace mongo_db { result_type operator()(const chain_properties_update_operation& op); result_type operator()(const delegation_reward_operation& op); result_type operator()(const auction_window_reward_operation& op); + result_type operator()(const total_comment_reward_operation& op); }; }}} // golos::plugins::mongo_db diff --git a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp index 01c75fe43f..5133c2a29d 100644 --- a/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp +++ b/plugins/mongo_db/include/golos/plugins/mongo_db/mongo_db_state.hpp @@ -88,6 +88,7 @@ namespace mongo_db { result_type operator()(const chain_properties_update_operation& op); result_type operator()(const delegation_reward_operation& op); result_type operator()(const auction_window_reward_operation& op); + result_type operator()(const total_comment_reward_operation& op); void write_global_property_object(const dynamic_global_property_object& dgpo, bool history); diff --git a/plugins/mongo_db/mongo_db_operations.cpp b/plugins/mongo_db/mongo_db_operations.cpp index 4cbb08030d..d7261271b3 100644 --- a/plugins/mongo_db/mongo_db_operations.cpp +++ b/plugins/mongo_db/mongo_db_operations.cpp @@ -768,4 +768,9 @@ namespace mongo_db { return body; } + auto operation_writer::operator()(const total_comment_reward_operation& op) -> result_type { + result_type body; + return body; + } + }}} diff --git a/plugins/mongo_db/mongo_db_state.cpp b/plugins/mongo_db/mongo_db_state.cpp index 681650e3e1..52ff5844d6 100644 --- a/plugins/mongo_db/mongo_db_state.cpp +++ b/plugins/mongo_db/mongo_db_state.cpp @@ -1927,4 +1927,8 @@ namespace mongo_db { } + auto state_writer::operator()(const total_comment_reward_operation& op) -> result_type { + + } + }}} \ No newline at end of file diff --git a/plugins/operation_dump/CMakeLists.txt b/plugins/operation_dump/CMakeLists.txt new file mode 100644 index 0000000000..398d552381 --- /dev/null +++ b/plugins/operation_dump/CMakeLists.txt @@ -0,0 +1,44 @@ +set(CURRENT_TARGET operation_dump) + +list(APPEND CURRENT_TARGET_HEADERS + include/golos/plugins/operation_dump/operation_dump_plugin.hpp + include/golos/plugins/operation_dump/operation_dump_container.hpp +) + +list(APPEND CURRENT_TARGET_SOURCES + operation_dump_plugin.cpp +) + +if(BUILD_SHARED_LIBRARIES) + add_library(golos_${CURRENT_TARGET} SHARED + ${CURRENT_TARGET_HEADERS} + ${CURRENT_TARGET_SOURCES} + ) +else() + add_library(golos_${CURRENT_TARGET} STATIC + ${CURRENT_TARGET_HEADERS} + ${CURRENT_TARGET_SOURCES} + ) +endif() + +add_library(golos::${CURRENT_TARGET} ALIAS golos_${CURRENT_TARGET}) +set_property(TARGET golos_${CURRENT_TARGET} PROPERTY EXPORT_NAME ${CURRENT_TARGET}) + +target_link_libraries( + golos_${CURRENT_TARGET} + golos::chain_plugin + golos::follow + golos::tags + appbase +) + +target_include_directories(golos_${CURRENT_TARGET} + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") + +install(TARGETS + golos_${CURRENT_TARGET} + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_container.hpp b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_container.hpp new file mode 100644 index 0000000000..c89a6d7fce --- /dev/null +++ b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_container.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +namespace golos { namespace plugins { namespace operation_dump { + +namespace bfs = boost::filesystem; + +// Structure size can differ - uses sizeof +struct dump_header { + char magic[13] = "Golos\adumpOP"; + uint32_t version = 1; +}; + +using operation_number = std::pair; + +class dump_buffer : public std::stringstream { +public: + dump_buffer() { + } + + using std::stringstream::write; + + void write(const operation_number& op_num) { + fc::raw::pack(*this, op_num); + } +}; + +using dump_buffers = std::map; + +class dump_file : public bfs::ofstream { +public: + dump_file(const bfs::path& p): bfs::ofstream(p, std::ios_base::binary | std::ios_base::app) { + bfs::ofstream::exceptions(std::ofstream::failbit | std::ofstream::badbit); + } + + void write(const dump_header& hdr) { + bfs::ofstream::write((const char*)&hdr, sizeof(dump_header)); + } +}; + +} } } // golos::plugins::operation_dump diff --git a/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_plugin.hpp b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_plugin.hpp new file mode 100644 index 0000000000..51e0259a91 --- /dev/null +++ b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_plugin.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include + +namespace golos { namespace plugins { namespace operation_dump { + +namespace bpo = boost::program_options; +using namespace golos::chain; + +class operation_dump_plugin final : public appbase::plugin { +public: + APPBASE_PLUGIN_REQUIRES((chain::plugin)) + + operation_dump_plugin(); + + ~operation_dump_plugin(); + + void set_program_options(bpo::options_description& cli, bpo::options_description& cfg) override; + + void plugin_initialize(const bpo::variables_map& options) override; + + void plugin_startup() override; + + void plugin_shutdown() override; + + static const std::string& name(); + +private: + class operation_dump_plugin_impl; + + std::unique_ptr my; +}; + +} } } //golos::plugins::operation_dump diff --git a/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_visitor.hpp b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_visitor.hpp new file mode 100644 index 0000000000..4c053a2363 --- /dev/null +++ b/plugins/operation_dump/include/golos/plugins/operation_dump/operation_dump_visitor.hpp @@ -0,0 +1,184 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace golos { namespace plugins { namespace operation_dump { + +using namespace golos::plugins::follow; + +#define COMMENT_ID(OP) hash_id(std::string(OP.author) + "/" + OP.permlink) + +#define TAGS_NUMBER 15 +#define TAG_MAX_LENGTH 512 + +class operation_dump_visitor { +public: + using result_type = void; + + dump_buffers& _buffers; + + const signed_block& _block; + uint16_t& _op_in_block; + + database& _db; + + operation_dump_visitor(dump_buffers& buffers, const signed_block& block, uint16_t& op_in_block, database& db) + : _buffers(buffers), _block(block), _op_in_block(op_in_block), _db(db) { + } + + uint64_t hash_id(const std::string& id) { + return fc::hash64(id.c_str(), id.length()); + } + + dump_buffer& write_op_header(const std::string& file_name, uint64_t op_related_id) { + auto& b = _buffers[file_name]; + b.write(operation_number(_block.block_num(), _op_in_block)); + fc::raw::pack(b, op_related_id); + return b; + } + + template + auto operator()(const T&) -> result_type { + } + + auto operator()(const transfer_operation& op) -> result_type { + auto& b = write_op_header("transfers", 0); + + fc::raw::pack(b, op); + } + + auto operator()(const comment_operation& op) -> result_type { + auto& b = write_op_header("comments", COMMENT_ID(op)); + + fc::raw::pack(b, op.parent_author); + fc::raw::pack(b, op.parent_permlink); + fc::raw::pack(b, op.author); + fc::raw::pack(b, op.permlink); + fc::raw::pack(b, op.title); + fc::raw::pack(b, op.body); + + auto meta = golos::plugins::tags::get_metadata(op.json_metadata, TAGS_NUMBER, TAG_MAX_LENGTH); + fc::raw::pack(b, meta); + } + + auto operator()(const delete_comment_operation& op) -> result_type { + if (_db.find_comment(op.author, op.permlink)) { + return; + } + + write_op_header("delete_comments", COMMENT_ID(op)); + } + + auto operator()(const comment_benefactor_reward_operation& op) -> result_type { + auto& b = write_op_header("benefactor_rewards", COMMENT_ID(op)); + + fc::raw::pack(b, op); + } + + auto operator()(const author_reward_operation& op) -> result_type { + auto& b = write_op_header("author_rewards", COMMENT_ID(op)); + + fc::raw::pack(b, op); + } + + auto operator()(const curation_reward_operation& op) -> result_type { + auto& b = write_op_header("curation_rewards", hash_id(std::string(op.comment_author) + "/" + op.comment_permlink)); + + fc::raw::pack(b, op); + } + + auto operator()(const auction_window_reward_operation& op) -> result_type { + auto& b = write_op_header("auction_window_rewards", hash_id(std::string(op.comment_author) + "/" + op.comment_permlink)); + + fc::raw::pack(b, op); + } + + auto operator()(const total_comment_reward_operation& op) -> result_type { + auto& b = write_op_header("total_comment_rewards", COMMENT_ID(op)); + + fc::raw::pack(b, op); + } + + auto operator()(const vote_operation& op) -> result_type { + auto& b = write_op_header("votes", COMMENT_ID(op)); + + fc::raw::pack(b, op); + + fc::raw::pack(b, _block.timestamp); + } + + // Not logs if operation failed in plugin, but logs if plugin not exists + auto operator()(const custom_json_operation& op) -> result_type { + if (op.id != "follow") { // follows, reblogs, delete_reblogs + return; + } + + std::vector fpops; + + auto v = fc::json::from_string(op.json); + try { + if (v.is_array() && v.size() > 0 && v.get_array()[0].is_array()) { + fc::from_variant(v, fpops); + } else { + fpops.emplace_back(); + fc::from_variant(v, fpops[0]); + } + } catch (...) { + // Normal cases failed, try this strange case from follow-plugin + try { + auto fop = v.as(); + fpops.emplace_back(fop); + } catch (...) { + } + } + + for (const follow_plugin_operation& fpop : fpops) { + fpop.visit(*this); + } + } + + auto operator()(const follow_operation& op) -> result_type { + auto& b = write_op_header("follows", hash_id(std::string(op.follower) + "/" + op.following)); + + fc::raw::pack(b, op.follower); + fc::raw::pack(b, op.following); + + uint16_t what = 0; + for (const auto& target : op.what) { + if (target == "blog") { + what |= 1 << blog; + } else if (target == "ignore") { + what |= 1 << ignore; + } + } + fc::raw::pack(b, what); + } + + auto operator()(const reblog_operation& op) -> result_type { + auto& b = write_op_header("reblogs", COMMENT_ID(op)); + + fc::raw::pack(b, op.account); + fc::raw::pack(b, op.author); + fc::raw::pack(b, op.permlink); + fc::raw::pack(b, op.title); // Usually empty, but no problems to dump + fc::raw::pack(b, op.body); + // Usually it is: + // {"app": "golos.io/0.1", "format": "text"} + // Not seems to be need + //fc::raw::pack(b, op.json_metadata); + + fc::raw::pack(b, _block.timestamp); + } + + auto operator()(const delete_reblog_operation& op) -> result_type { + auto& b = write_op_header("delete_reblogs", COMMENT_ID(op)); + + fc::raw::pack(b, op.account); + } +}; + +} } } // golos::plugins::operation_dump diff --git a/plugins/operation_dump/operation_dump_plugin.cpp b/plugins/operation_dump/operation_dump_plugin.cpp new file mode 100644 index 0000000000..7c40eb5fec --- /dev/null +++ b/plugins/operation_dump/operation_dump_plugin.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include + +namespace golos { namespace plugins { namespace operation_dump { + +namespace bfs = boost::filesystem; + +class operation_dump_plugin::operation_dump_plugin_impl final { +public: + operation_dump_plugin_impl() + : _db(appbase::app().get_plugin().db()) { + } + + ~operation_dump_plugin_impl() { + } + + void on_block(const signed_block& block) { + auto lib = _db.last_non_undoable_block_num(); + + for (auto block_num = start_block; block_num <= lib; ++block_num) { + auto block = _db.get_block_log().read_block_by_num(block_num); + if (!block) { + return; + } + + try { + uint16_t op_in_block = 0; + + operation_dump_visitor op_visitor(buffers, *block, op_in_block, _db); + + for (const auto& trx : block->transactions) { + for (const auto& op : trx.operations) { + op.visit(op_visitor); + ++op_in_block; + } + } + + for (const auto& op : virtual_ops[block_num]) { + op.visit(op_visitor); + ++op_in_block; + } + } catch (...) { + virtual_ops.erase(block_num); + start_block = block_num+1; + throw; + } + + virtual_ops.erase(block_num); + } + + start_block = lib+1; + + for (auto& it : buffers) { + bfs::create_directories(operation_dump_dir); + dump_file file(operation_dump_dir / it.first); + if (file.tellp() == 0) { + file.write(dump_header()); + } + file << it.second.rdbuf(); + } + buffers.clear(); + } + + void on_operation(const operation_notification& note) { + if (!is_virtual_operation(note.op)) { + return; + } + + virtual_ops[note.block].push_back(note.op); + + // remove ops if there were forks and rollbacks + auto itr = virtual_ops.find(note.block); + ++itr; + virtual_ops.erase(itr, virtual_ops.end()); + } + + database& _db; + + bfs::path operation_dump_dir; + + uint32_t start_block = 1; + + std::map> virtual_ops; + dump_buffers buffers; +}; + +operation_dump_plugin::operation_dump_plugin() = default; + +operation_dump_plugin::~operation_dump_plugin() = default; + +const std::string& operation_dump_plugin::name() { + static std::string name = "operation_dump"; + return name; +} + +void operation_dump_plugin::set_program_options(bpo::options_description& cli, bpo::options_description& cfg) { + cfg.add_options() ( + "operation-dump-dir", bpo::value()->default_value("operation_dump"), + "The location of the dir to dump operations to (abs path or relative to application data dir)." + ); +} + +void operation_dump_plugin::plugin_initialize(const bpo::variables_map& options) { + ilog("Initializing operation dump plugin"); + + my = std::make_unique(); + + auto odd = options.at("operation-dump-dir").as(); + if (odd.is_relative()) { + my->operation_dump_dir = appbase::app().data_dir() / odd; + } else { + my->operation_dump_dir = odd; + } + + my->_db.applied_block.connect([&](const signed_block& b) { + my->on_block(b); + }); + + my->_db.post_apply_operation.connect([&](const operation_notification& note) { + my->on_operation(note); + }); +} + +void operation_dump_plugin::plugin_startup() { + ilog("Starting up operation dump plugin"); +} + +void operation_dump_plugin::plugin_shutdown() { + ilog("Shutting down operation dump plugin"); +} + +} } } // golos::plugins::operation_dump diff --git a/programs/golosd/CMakeLists.txt b/programs/golosd/CMakeLists.txt index d02b81f5de..81895cbb1f 100644 --- a/programs/golosd/CMakeLists.txt +++ b/programs/golosd/CMakeLists.txt @@ -28,6 +28,7 @@ target_link_libraries( golos::social_network golos::tags golos::market_history + golos::operation_dump golos::operation_history golos::statsd golos::account_by_key diff --git a/programs/golosd/main.cpp b/programs/golosd/main.cpp index f855e59484..c6ea4a4e81 100644 --- a/programs/golosd/main.cpp +++ b/programs/golosd/main.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #ifdef MONGODB_PLUGIN_BUILT #include #endif @@ -81,6 +82,7 @@ namespace golos { appbase::app().register_plugin(); appbase::app().register_plugin(); appbase::app().register_plugin(); + appbase::app().register_plugin(); #ifdef MONGODB_PLUGIN_BUILT appbase::app().register_plugin(); #endif diff --git a/share/golosd/config/config.ini b/share/golosd/config/config.ini index f756fc3beb..f0c24505ac 100644 --- a/share/golosd/config/config.ini +++ b/share/golosd/config/config.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 2G # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 1000 # each 3000 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_history account_history account_notes statsd block_info raw_block witness_api +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_dump operation_history account_history account_notes statsd block_info raw_block witness_api # Remove votes before defined block, should increase performance clear-votes-before-block = 4294967295 # clear votes after each cashout diff --git a/share/golosd/config/config_debug.ini b/share/golosd/config/config_debug.ini index b3293e7df6..2a5d74e4f9 100644 --- a/share/golosd/config/config_debug.ini +++ b/share/golosd/config/config_debug.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 100M # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 10 # each 30 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_history statsd block_info raw_block debug_node witness_api +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_dump operation_history statsd block_info raw_block debug_node witness_api # Remove votes before defined block, should increase performance clear-votes-before-block = 0 # don't clear votes diff --git a/share/golosd/config/config_debug_mongo.ini b/share/golosd/config/config_debug_mongo.ini index b144cf19be..d459b6b73d 100644 --- a/share/golosd/config/config_debug_mongo.ini +++ b/share/golosd/config/config_debug_mongo.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 100M # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 10 # each 30 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_history statsd block_info raw_block debug_node witness_api mongo_db +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key account_history account_notes operation_dump operation_history statsd block_info raw_block debug_node witness_api mongo_db # For connect to mongodb which is running outside Docker (if golosd running inside) mongodb-uri = mongodb://172.17.0.1:27017/Golos diff --git a/share/golosd/config/config_mongo.ini b/share/golosd/config/config_mongo.ini index 9d477202e9..552ae7ee1e 100644 --- a/share/golosd/config/config_mongo.ini +++ b/share/golosd/config/config_mongo.ini @@ -66,7 +66,7 @@ inc-shared-file-size = 2G # and resizes. The optimal strategy is do checking of the free space, but not very often. block-num-check-free-size = 1000 # each 3000 seconds -plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_history account_history account_notes statsd block_info raw_block witness_api mongo_db +plugin = chain p2p json_rpc webserver network_broadcast_api witness test_api database_api private_message follow social_network tags market_history account_by_key operation_dump operation_history account_history account_notes statsd block_info raw_block witness_api mongo_db # For connect to mongodb which is running outside Docker (if golosd running inside) mongodb-uri = mongodb://172.17.0.1:27017/Golos