Merge branch 'develop' into bug/IS-862-swt-grow

skalenetwork · Sep 22, 2023 · 0752229 · 0752229
2 parents 696fc12 + 76ca0f4
commit 0752229
Show file tree

Hide file tree

Showing 12 changed files with 124 additions and 54 deletions.
diff --git a/README.md b/README.md
@@ -58,11 +58,15 @@ If you have already cloned the repo and forgot to pass `--recurse-submodules`, e
 
 ```
 sudo apt update
-sudo apt install autoconf build-essential cmake libprocps-dev libtool texinfo wget yasm flex bison btrfs-progs 
-sudo apt install make build-essential cmake pkg-config libgnutls28-dev libssl-dev unzip zlib1g-dev libgcrypt20-dev docker.io gcc-9 g++-9 gperf clang-format-11
+sudo apt install autoconf build-essential cmake libprocps-dev libtool texinfo wget yasm flex bison btrfs-progs python python3-pip gawk git vim doxygen 
+sudo apt install make build-essential cmake pkg-config libgnutls28-dev libssl-dev unzip zlib1g-dev libgcrypt20-dev docker.io gcc-9 g++-9 gperf clang-format-11 gnutls-dev
+sudo apt install nettle-dev libhiredis-dev redis-server google-perftools libgoogle-perftools-dev lcov
 ```
 
-NB cmake needs to be of version >=3.31, git of version >=2.18
+
+
+
+NB cmake needs to be of version >=3.21, git of version >=2.18
 
 ### (for Ubuntu 20.10 or later) Set  gcc-9 as default compiler
 ```
@@ -74,14 +78,15 @@ sudo update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-to
 gcc --version
 ```
 
-### Build dependencies
+# Install latest cmake 
 
 ```
-cd deps
-./build.sh
+sudo apt-get purge cmake
+sudo snap install cmake --classic
 ```
 
-or, if you want to build debug version of skaled
+
+### Build dependencies
 
 ```
 cd deps
@@ -107,8 +112,6 @@ cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Debug
 cmake --build build -- -j$(nproc)
 ```
 
-Note: Currently only Debug build is supported.
-
 
 ## Testing
 

diff --git a/deps/build.sh b/deps/build.sh
@@ -703,7 +703,7 @@ if [ "$WITH_UNWIND" = "yes" ];
             #eval autoheader
             #eval automake --add-missing
             eval autoreconf -i
-            eval ./configure --disable-shared --prefix="$INSTALL_ROOT"
+            eval ./configure --disable-shared --disable-tests --prefix="$INSTALL_ROOT"
             cd ..
         fi
         echo -e "${COLOR_INFO}building it${COLOR_DOTS}...${COLOR_RESET}"
@@ -932,13 +932,8 @@ then
 		cd "$SOURCES_ROOT"
 		if [ ! -d "libiconv-1.15" ];
 		then
-			if [ ! -f "libiconv-1.15.tar.gz" ];
-			then
-				echo -e "${COLOR_INFO}downloading it${COLOR_DOTS}...${COLOR_RESET}"
-				eval "$WGET" --no-check-certificate https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz
-			fi
 			echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"
-			eval tar -xzf libiconv-1.15.tar.gz
+			eval tar -xzf "$PREDOWNLOADED_ROOT/libiconv-1.15.tar.gz"
 			echo -e "${COLOR_INFO}configuring it${COLOR_DOTS}...${COLOR_RESET}"
 			cd libiconv-1.15
 			eval ./configure "${CONF_CROSSCOMPILING_OPTS_GENERIC}" --enable-static --disable-shared --prefix="$INSTALL_ROOT" "${CONF_DEBUG_OPTIONS}"
@@ -1113,7 +1108,7 @@ then
 			#		..
 			#cd ../..
 			eval ./autogen.sh
-			eval ./configure "${CONF_CROSSCOMPILING_OPTS_GENERIC}" --enable-static --disable-shared --prefix="$INSTALL_ROOT" "${CONF_DEBUG_OPTIONS}"
+                        eval ./configure "${CONF_CROSSCOMPILING_OPTS_GENERIC}" --enable-static --disable-shared --disable-samples --prefix="$INSTALL_ROOT" "${CONF_DEBUG_OPTIONS}"
 			cd ..
 		fi
 		#cd libevent/build
@@ -1142,7 +1137,7 @@ then
 			if [ ! -f "libuv-from-git.tar.gz" ];
 			then
 				echo -e "${COLOR_INFO}getting it from git${COLOR_DOTS}...${COLOR_RESET}"
-                eval git clone https://github.com/libuv/libuv.git
+                                eval git clone https://github.com/libuv/libuv.git
 				cd libuv
 				eval git checkout v1.x
 				eval git pull
@@ -1191,11 +1186,11 @@ then
 			if [ ! -f "libwebsockets-from-git.tar.gz" ];
 			then
 				echo -e "${COLOR_INFO}downloading it${COLOR_DOTS}...${COLOR_RESET}"
-                eval git clone https://github.com/warmcat/libwebsockets.git
+                                eval git clone https://github.com/warmcat/libwebsockets.git
 				eval cd libwebsockets
-                # eval git checkout v4.1-stable
-                eval git checkout v4.3-stable
-                eval git pull
+                                # eval git checkout v4.1-stable
+                                eval git checkout v4.3-stable
+                                eval git pull
 				cd ..
 				echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
 				eval tar -czf libwebsockets-from-git.tar.gz ./libwebsockets
@@ -1391,7 +1386,7 @@ then
 				eval "$WGET" https://boostorg.jfrog.io/artifactory/main/release/1.68.0/source/boost_1_68_0.tar.bz2
 			fi
 			echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"
-            eval tar -xf boost_1_68_0.tar.bz2
+                        eval tar -xf boost_1_68_0.tar.bz2
 		fi
 		cd boost_1_68_0
 		echo -e "${COLOR_INFO}configuring and building it${COLOR_DOTS}...${COLOR_RESET}"
@@ -1996,10 +1991,10 @@ then
 			then
 				echo -e "${COLOR_INFO}getting it from git${COLOR_DOTS}...${COLOR_RESET}"
 				eval git clone https://github.com/google/glog.git --recursive
-                cd glog
-                eval git checkout ee6faf13b20de9536f456bd84584f4ab4db1ceb4
-                cd ..
-                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
+                                cd glog
+                                eval git checkout ee6faf13b20de9536f456bd84584f4ab4db1ceb4
+                                cd ..
+                                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
 				eval tar -czf glog-from-git.tar.gz ./glog
 			else
 				echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"
@@ -2010,7 +2005,7 @@ then
 			eval mkdir -p build
 			cd build
 			eval "$CMAKE" "${CMAKE_CROSSCOMPILING_OPTS}" -DCMAKE_INSTALL_PREFIX="$INSTALL_ROOT" -DCMAKE_BUILD_TYPE="$TOP_CMAKE_BUILD_TYPE" \
-				-DBUILD_SHARED_LIBS=OFF -DWITH_UNWIND=OFF \
+                                -DBUILD_SHARED_LIBS=OFF -DWITH_UNWIND=OFF -DWITH_GTEST=OFF \
 				..
 			cd ..
 		else
@@ -2077,11 +2072,11 @@ then
 			then
 				echo -e "${COLOR_INFO}getting it from git${COLOR_DOTS}...${COLOR_RESET}"
 				eval git clone https://github.com/facebook/folly.git --recursive
-                cd folly
-                eval git checkout 5c8fc1b622422a1c73f46d6fb51ac1164d8efb0f
-                cd ..
-                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
-				eval tar -czf folly-from-git.tar.gz ./folly
+                                cd folly
+                                eval git checkout 5c8fc1b622422a1c73f46d6fb51ac1164d8efb0f
+                                cd ..
+                                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
+                                eval tar -czf folly-from-git.tar.gz ./folly
 			else
 				echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"
 				eval tar -xzf folly-from-git.tar.gz
@@ -2165,10 +2160,10 @@ then
 			then
 				echo -e "${COLOR_INFO}getting it from git${COLOR_DOTS}...${COLOR_RESET}"
 				eval git clone https://github.com/google/googletest.git --recursive
-                cd googletest
-                eval git checkout 4c5650f68866e3c2e60361d5c4c95c6f335fb64b
-                cd ..
-                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
+                                cd googletest
+                                eval git checkout 4c5650f68866e3c2e60361d5c4c95c6f335fb64b
+                                cd ..
+                                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
 				eval tar -czf gtest-from-git.tar.gz ./googletest
 			else
 				echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"
@@ -2208,10 +2203,10 @@ then
 			then
 				echo -e "${COLOR_INFO}getting it from git${COLOR_DOTS}...${COLOR_RESET}"
 				eval git clone https://github.com/facebookincubator/fizz.git --recursive
-                cd fizz
-                eval git checkout 93003f4161f7cebe1c121b3232215db8314c2ce7
-                cd ..
-                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
+                                cd fizz
+                                eval git checkout 93003f4161f7cebe1c121b3232215db8314c2ce7
+                                cd ..
+                                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
 				eval tar -czf fizz-from-git.tar.gz ./fizz
 			else
 				echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"
@@ -2253,10 +2248,10 @@ then
 			then
 				echo -e "${COLOR_INFO}getting it from git${COLOR_DOTS}...${COLOR_RESET}"
 				eval git clone https://github.com/facebook/wangle.git --recursive
-                cd wangle
-                eval git checkout 7249d3f8d18bcd4bc13649d13654ccb2a771f7b3
-                cd ..
-                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
+                                cd wangle
+                                eval git checkout 7249d3f8d18bcd4bc13649d13654ccb2a771f7b3
+                                cd ..
+                                echo -e "${COLOR_INFO}archiving it${COLOR_DOTS}...${COLOR_RESET}"
 				eval tar -czf wangle-from-git.tar.gz ./wangle
 			else
 				echo -e "${COLOR_INFO}unpacking it${COLOR_DOTS}...${COLOR_RESET}"

diff --git a/deps/pre_downloaded/libiconv-1.15.tar.gz b/deps/pre_downloaded/libiconv-1.15.tar.gz
diff --git a/libdevcore/Common.cpp b/libdevcore/Common.cpp
@@ -165,8 +165,8 @@ void ExitHandler::exitHandler( int nSignalNo, ExitHandler::exit_code_t ec ) {
         s_ec = ec;
     }
 
-    // indicate failure if signal is not INT or TERM!
-    if ( s_ec == ec_success && nSignalNo != SIGINT && nSignalNo != SIGTERM )
+    // indicate failure if signal is not INT or TERM or internal (-1)
+    if ( s_ec == ec_success && nSignalNo > 0 && nSignalNo != SIGINT && nSignalNo != SIGTERM )
         s_ec = ExitHandler::ec_failure;
 
     s_bStop = true;

diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp
@@ -1263,7 +1263,8 @@ void BlockChain::garbageCollect( bool _force ) {
 
     m_lastCollection = chrono::system_clock::now();
 
-    while ( m_lastStats.memTotal() >= c_maxCacheSize ) {
+    // We subtract memory that blockhashes occupy because it is treated sepaparately
+    while ( m_lastStats.memTotal() - m_lastStats.memBlockHashes >= c_maxCacheSize ) {
         Guard l( x_cacheUsage );
         for ( CacheID const& id : m_cacheUsage.back() ) {
             m_inUse.erase( id );
@@ -1315,6 +1316,7 @@ void BlockChain::garbageCollect( bool _force ) {
 
     {
         WriteGuard l( x_blockHashes );
+        // This is where block hash memory cleanup is treated
         // allow only 4096 blockhashes in the cache
         if ( m_blockHashes.size() > 4096 ) {
             auto last = m_blockHashes.begin();

diff --git a/libethereum/ClientBase.cpp b/libethereum/ClientBase.cpp
@@ -132,7 +132,7 @@ std::pair< u256, ExecutionResult > ClientBase::estimateGas( Address const& _from
             estimateGasStep( upperBound, bk, _from, _dest, _value, gasPrice, _data );
         if ( estimatedStep.first ) {
             auto executionResult = estimatedStep.second;
-            auto gasUsed = executionResult.gasUsed.convert_to< int64_t >();
+            auto gasUsed = std::max( executionResult.gasUsed.convert_to< int64_t >(), lowerBound );
 
             estimatedStep = estimateGasStep( gasUsed, bk, _from, _dest, _value, gasPrice, _data );
             if ( estimatedStep.first ) {

diff --git a/libskale/State.cpp b/libskale/State.cpp
@@ -801,8 +801,19 @@ void State::clearStorage( Address const& _contract ) {
     for ( auto const& hashPairPair : storage_WITHOUT_LOCK( _contract ) ) {
         auto const& key = hashPairPair.second.first;
         auto const& value = hashPairPair.second.first;
+        // Set storage to zero in state cache
         clearStorageValue( _contract, key, value );
+        // Set storage to zero in the account storage cache
+        // we have lots of caches, some of them may be unneeded
+        // will analyze this more in future releases
         acc->setStorageCache( key, 0 );
+        /* The corresponding key/value pair needs to be cleared in database
+           Inserting ZERO deletes the key during commit
+           at the end of transaction
+           see OverlayDB::commitStorageValues()
+        */
+        h256 ZERO( 0 );
+        m_db_ptr->insert( _contract, key, ZERO );
     }
 
     totalStorageUsed_ -= ( accStorageUsed + storageUsage[_contract] );

diff --git a/libskutils/include/skutils/stats.h b/libskutils/include/skutils/stats.h
@@ -220,7 +220,7 @@ class element : public skutils::ref_retain_release {
     element( const char* strSubSystem, const char* strProtocol, const char* strMethod,
         int /*nServerIndex*/, int /*ipVer*/ );
     virtual ~element();
-    void stop() const;
+    void stop();
     void setMethod( const char* strMethod ) const;
     void setError() const;
     double getDurationInSeconds() const;

diff --git a/libskutils/src/stats.cpp b/libskutils/src/stats.cpp
@@ -405,7 +405,6 @@ element::element( const char* strSubSystem, const char* strProtocol, const char*
         strProtocol_ = "N/A";
     if ( strMethod_.empty() )
         strMethod_ = g_strMethodNameUnknown;
-    do_register();
 }
 element::~element() {
     stop();
@@ -420,12 +419,13 @@ void element::do_unregister() {
     queue::getQueueForSubsystem( strSubSystem_.c_str() ).do_unregister( rttElement );
 }
 
-void element::stop() const {
+void element::stop() {
     lock_type lock( mtx() );
     if ( isStopped_ )
         return;
     isStopped_ = true;
     tpEnd_ = skutils::stats::clock::now();
+    do_register();
 }
 
 void element::setMethod( const char* strMethod ) const {

diff --git a/skaled/CMakeLists.txt b/skaled/CMakeLists.txt
@@ -25,8 +25,8 @@ target_link_libraries(
         devcore
         Boost::program_options
         skutils
-        "${DEPS_INSTALL_ROOT}/lib/liblzma.a"
         "${DEPS_INSTALL_ROOT}/lib/libunwind.a"
+        "${DEPS_INSTALL_ROOT}/lib/liblzma.a"
         pthread
         idn2
         batched-io

diff --git a/storage_benchmark/CMakeLists.txt b/storage_benchmark/CMakeLists.txt
@@ -20,8 +20,8 @@ target_link_libraries(
         historic
         skutils
         devcore
-        "${DEPS_INSTALL_ROOT}/lib/liblzma.a"
         "${DEPS_INSTALL_ROOT}/lib/libunwind.a"
+        "${DEPS_INSTALL_ROOT}/lib/liblzma.a"
     )
 
 #target_include_directories(evm_benchmark PRIVATE ../utils)

diff --git a/test/unittests/libweb3jsonrpc/jsonrpc.cpp b/test/unittests/libweb3jsonrpc/jsonrpc.cpp
@@ -1806,6 +1806,65 @@ contract Logger{
     BOOST_REQUIRE_EQUAL(logs.size(), 24);
 }
 
+BOOST_AUTO_TEST_CASE( estimate_gas_low_gas_txn ) {
+    JsonRpcFixture fixture;
+    dev::eth::simulateMining( *( fixture.client ), 10 );
+
+    auto senderAddress = fixture.coinbase.address();
+
+/*
+// SPDX-License-Identifier: None
+pragma solidity ^0.6.0;
+
+contract TestEstimateGas {
+    uint256[256] number;
+    uint256 counter = 0;
+
+    function store(uint256 x) public {
+        number[counter] = x;
+        counter += 1;
+    }
+
+    function clear(uint256 pos) public {
+        number[pos] = 0;
+    }
+}
+*/
+
+    string bytecode = "608060405260006101005534801561001657600080fd5b50610104806100266000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c80636057361d146037578063c0fe1af8146062575b600080fd5b606060048036036020811015604b57600080fd5b8101908080359060200190929190505050608d565b005b608b60048036036020811015607657600080fd5b810190808035906020019092919050505060b8565b005b806000610100546101008110609e57fe5b018190555060016101006000828254019250508190555050565b60008082610100811060c657fe5b01819055505056fea26469706673582212206c8da972693a5b8c9bf59c197c4a0c554e9f51abd20047572c9c19125b533d2964736f6c634300060c0033";
+
+    Json::Value create;
+    create["code"] = bytecode;
+    create["gas"] = "180000";  // TODO or change global default of 90000?
+
+    string deployHash = fixture.rpcClient->eth_sendTransaction( create );
+    dev::eth::mineTransaction( *( fixture.client ), 1 );
+
+    Json::Value deployReceipt = fixture.rpcClient->eth_getTransactionReceipt( deployHash );
+    string contractAddress = deployReceipt["contractAddress"].asString();
+
+    Json::Value txStore1;  // call store(1)
+    txStore1["to"] = contractAddress;
+    txStore1["data"] = "0x6057361d0000000000000000000000000000000000000000000000000000000000000001";
+    txStore1["from"] = toJS( senderAddress );
+    txStore1["gasPrice"] = fixture.rpcClient->eth_gasPrice();
+    string txHash = fixture.rpcClient->eth_call( txStore1, "latest" );
+
+    Json::Value estimateGasCall;  // call clear(0)
+    estimateGasCall["to"] = contractAddress;
+    estimateGasCall["data"] = "0xc0fe1af80000000000000000000000000000000000000000000000000000000000000000";
+    estimateGasCall["from"] = toJS( senderAddress );
+    estimateGasCall["gasPrice"] = fixture.rpcClient->eth_gasPrice();
+    string estimatedGas = fixture.rpcClient->eth_estimateGas( estimateGasCall );
+
+    dev::bytes data = dev::jsToBytes( estimateGasCall["data"].asString() );
+
+    BOOST_REQUIRE( dev::jsToU256( estimatedGas ) > dev::eth::TransactionBase::baseGasRequired(
+                       false, &data, fixture.client->chainParams().scheduleForBlockNumber(
+                           fixture.client->number() ) ) );
+    BOOST_REQUIRE( dev::jsToU256( estimatedGas ) == 21871 );
+}
+
 BOOST_AUTO_TEST_CASE( storage_limit_contract ) {
     JsonRpcFixture fixture;
     dev::eth::simulateMining( *( fixture.client ), 10 );