Fixed repetetive objects loading from ipfs (#513)

* Fixed unnecessary ram consumption * Light refactor of upgrade commit * Copy changes to v5 * fixed repetetive objects loading from ipfs * Fixed upgrade over 1 version * Fixed test * ci tests * fix typo * fix a bug * Fix test13 * build all versions * Increased version --------- Co-authored-by: SilkovAlexander <[email protected]> Co-authored-by: SilkovAlexander <[email protected]> Co-authored-by: Vasily Selivanov <[email protected]>
gosh-sh · Jun 24, 2023 · 9341667 · 9341667
1 parent 2e52e2a
commit 9341667
Show file tree

Hide file tree

Showing 60 changed files with 1,043 additions and 577 deletions.
diff --git a/.ci/Jenkinsfile.gosh-prepare-rc b/.ci/Jenkinsfile.gosh-prepare-rc
@@ -113,7 +113,7 @@ pipeline {
                     sh "rm -rf tests_workspace_*"
                     def basePort = 9000
                     def files = getTestFiles("tests").toList()
-                    def versions = ["v1_x", "v2_x", "v3_x", "v4_x", "v5_x/v5.1.0", "v5_x/v5.0.0"]
+                    def versions = ["v4_x", "v5_x/v5.1.0", "v5_x/v5.0.0"]
                     def batches = []
                     for (int i = 0; i < files.size(); i += batchSize) {
                         def end = Math.min(i + batchSize, files.size())
@@ -126,10 +126,6 @@ pipeline {
                             sh "rm -rf ${version}_workspace_*"
                             deployVersion(version, batch, basePort + (j * batchSize))
                         }
-
-                        runFunctionalTests(batch, basePort + (j * batchSize), "Func v1", "v1_x")
-                        runFunctionalTests(batch, basePort + (j * batchSize), "Func v2", "v2_x")
-                        runFunctionalTests(batch, basePort + (j * batchSize), "Func v2", "v3_x")
                         runFunctionalTests(batch, basePort + (j * batchSize), "Func v4", "v4_x")
                         runFunctionalTests(batch, basePort + (j * batchSize), "Func v5.0", "v5_x/v5.0.0")
                         runFunctionalTests(batch, basePort + (j * batchSize), "Func v5.1", "v5_x/v5.1.0")
@@ -151,7 +147,7 @@ pipeline {
                 script {
                     sh "rm -rf tests_workspace_*"
                     def basePort = 9000
-                    def versions = ["v2_x", "v3_x", "v4_x", "v5_x/v5.0.0", "v5_x/v5.1.0"]
+                    def versions = ["v4_x", "v5_x/v5.0.0", "v5_x/v5.1.0"]
 
                     def upgradeFiles = getTestFiles("tests/upgrade_tests").toList()
                     def upgradeBatches = []
@@ -170,8 +166,6 @@ pipeline {
 
                         runUpgradeTests(batch, basePort + (j * batchSize), "v5_x/v5.0.0", "v5_x/v5.1.0")
                         runUpgradeTests(batch, basePort + (j * batchSize), "v4_x", "v5_x/v5.1.0")
-                        runUpgradeTests(batch, basePort + (j * batchSize), "v3_x", "v5_x/v5.1.0")
-                        runUpgradeTests(batch, basePort + (j * batchSize), "v2_x", "v5_x/v5.1.0")
                         sh 'docker ps -a --filter name=localnode-* -aq | grep -v "$(docker ps -a --filter name=localnode-9000 -q)" | xargs docker rm -f'
                     }
 

diff --git a/gosh-dispatcher/Cargo.toml b/gosh-dispatcher/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "git-remote-gosh"
-version = "5.1.24"
+version = "5.1.25"
 edition = "2021"
 keywords = ['git-remote', 'git', 'gosh']
 rust-version = "1.68"

diff --git a/tests/28-ipfs_repetitive_requests.test.sh b/tests/28-ipfs_repetitive_requests.test.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+set -o pipefail
+. ./util.sh
+set -x
+
+# Test 28 - ensure that content from ipfs loaded once per blob.
+
+if [[ "$VERSION" == *"v1_x"* ]]; then
+  echo "Test is ignored for v1"
+  exit 0
+fi
+
+REPO_NAME="repo28_$(date +%s)"
+
+[ -d $REPO_NAME ] && rm -rf $REPO_NAME
+
+deploy_repo
+REPO_ADDR=$(get_repo_addr)
+
+echo "***** awaiting repo deploy *****"
+wait_account_active $REPO_ADDR
+
+echo "***** cloning repo *****"
+git clone gosh://$SYSTEM_CONTRACT_ADDR/$DAO_NAME/$REPO_NAME
+
+#check
+cd $REPO_NAME
+
+# config git client
+git config user.email "[email protected]"
+git config user.name "My name"
+git branch -m main
+
+max_files=10
+file_name=big_file.dat
+
+for n in $(seq 1 $max_files); do
+    dd if=/dev/urandom of=$file_name bs=16K count=10
+    git add $file_name
+    git commit -m "main: update '$file_name' ($n)"
+    git push -u origin main
+    delay 10
+done
+
+cd ..
+
+GOSH_TRACE=5 git clone gosh://$SYSTEM_CONTRACT_ADDR/$DAO_NAME/$REPO_NAME "${REPO_NAME}-clone" &> trace-clone.log
+
+set +o pipefail
+cnt=$(grep -c "load_data_from_ipfs: ipfs_address=" trace-clone.log)
+
+if (( $cnt != $max_files )); then
+    echo "TEST FAILED"
+    exit 1
+fi
+
+echo "TEST SUCCEEDED"
diff --git a/tests/upgrade_tests/02_5-push_after_upgrade_over_version.test.sh.ignore b/tests/upgrade_tests/02_5-push_after_upgrade_over_version.test.sh.ignore
@@ -0,0 +1,112 @@
+#!/bin/bash
+set -e 
+set -o pipefail
+. ./util.sh
+
+set -x
+
+if [ "$1" = "ignore" ]; then
+  echo "Test $0 ignored"
+  exit 0
+fi
+
+REPO_NAME=upgrade_repo02_5
+DAO_NAME="dao-upgrade-test02_5_$(date +%s)"
+NEW_REPO_PATH=upgrade_repo02_5_v2
+
+# delete folders
+[ -d $REPO_NAME ] && rm -rf $REPO_NAME
+[ -d $NEW_REPO_PATH ] && rm -rf $NEW_REPO_PATH
+
+# deploy new DAO that will be upgraded
+deploy_DAO_and_repo
+
+export OLD_LINK="gosh://$SYSTEM_CONTRACT_ADDR/$DAO_NAME/$REPO_NAME"
+echo "OLD_LINK=$OLD_LINK"
+
+echo "***** cloning old version repo *****"
+git clone $OLD_LINK
+
+# check
+cd $REPO_NAME
+git config user.email "[email protected]"
+git config user.name "My name"
+git branch -m main
+
+# push 1 file
+echo "***** Pushing file to old repo *****"
+echo old_ver > 1.txt
+git add 1.txt
+git commit -m test
+git push -u origin main
+#git log
+
+cd ..
+
+echo "Upgrade DAO"
+upgrade_DAO
+
+echo "***** new repo02_5 deploy *****"
+tonos-cli call --abi $WALLET_ABI_1 --sign $WALLET_KEYS $WALLET_ADDR AloneDeployRepository \
+    "{\"nameRepo\":\"$REPO_NAME\",\"descr\":\"\",\"previous\":{\"addr\":\"$REPO_ADDR\", \"version\":\"$CUR_VERSION\"}}" || exit 1
+REPO_ADDR=$(tonos-cli -j run $SYSTEM_CONTRACT_ADDR_1 getAddrRepository "{\"name\":\"$REPO_NAME\",\"dao\":\"$DAO_NAME\"}" --abi $SYSTEM_CONTRACT_ABI_1 | sed -n '/value0/ p' | cut -d'"' -f 4)
+
+echo "***** awaiting repo deploy *****"
+wait_account_active $REPO_ADDR
+sleep 3
+
+tonos-cli callx --addr $WALLET_ADDR --abi $WALLET_ABI_1 --keys $WALLET_KEYS -m sendTokenToNewVersionAuto
+sleep 30
+
+tonos-cli callx --addr $WALLET_ADDR --abi $WALLET_ABI_1 --keys $WALLET_KEYS -m sendTokenToNewVersionAuto
+sleep 30
+
+tonos-cli callx --addr "$WALLET_ADDR" --abi "$WALLET_ABI_1" --keys "$WALLET_KEYS" -m lockVoting --amount 0
+
+sleep 30
+
+echo "Upgrade DAO"
+upgrade_DAO $TEST_VERSION2 $SYSTEM_CONTRACT_ADDR_2
+
+echo "***** new repo02_5 deploy *****"
+tonos-cli call --abi $WALLET_ABI_1 --sign $WALLET_KEYS $WALLET_ADDR AloneDeployRepository \
+    "{\"nameRepo\":\"$REPO_NAME\",\"descr\":\"\",\"previous\":{\"addr\":\"$REPO_ADDR\", \"version\":\"$TEST_VERSION1\"}}" || exit 1
+REPO_ADDR=$(tonos-cli -j run $SYSTEM_CONTRACT_ADDR_2 getAddrRepository "{\"name\":\"$REPO_NAME\",\"dao\":\"$DAO_NAME\"}" --abi $SYSTEM_CONTRACT_ABI_2 | sed -n '/value0/ p' | cut -d'"' -f 4)
+
+echo "***** awaiting repo deploy *****"
+wait_account_active $REPO_ADDR
+sleep 3
+
+export NEW_LINK="gosh://$SYSTEM_CONTRACT_ADDR_1/$DAO_NAME/$REPO_NAME"
+echo "NEW_LINK=$NEW_LINK"
+
+echo "***** push after upgrade *****"
+cd $REPO_NAME
+git fetch
+echo new_ver > 1.txt
+git add 1.txt
+git commit -m test2
+GOSH_TRACE=5 git push -u origin main &> ../trace_2_5.log
+
+wait_set_commit $REPO_ADDR main
+
+cd ..
+
+sleep 20
+
+echo "***** cloning repo with new link *****"
+git clone $NEW_LINK $NEW_REPO_PATH
+
+echo "***** push to new version *****"
+cd $NEW_REPO_PATH
+
+cur_ver=$(cat 1.txt)
+if [ $cur_ver != "new_ver" ]; then
+  echo "WRONG VERSION"
+  exit 1
+fi
+echo "GOOD VERSION"
+
+cd ..
+
+echo "TEST SUCCEEDED"
diff --git a/tests/upgrade_tests/set_up.sh b/tests/upgrade_tests/set_up.sh
@@ -86,6 +86,10 @@ if [ -n "$3" ]; then
   echo "Deploy giver for upgrade"
   cd $NEW_GOSH_PATH/../multisig
 
+  echo "" > Giver.addr
+  echo "" > Giver.network
+  echo "" > Giver.seed
+
   make generate-docker
   export GIVER_ADDR=`cat Giver.addr`
   echo "GIVER_ADDR = $GIVER_ADDR"

diff --git a/tests/util.sh b/tests/util.sh
@@ -18,7 +18,7 @@ function get_repo_addr {
     params=$(jq -n --arg name "${REPO_NAME}" --arg dao "${DAO_NAME}" '$ARGS.named')
 
     tonos-cli -j -u $NETWORK run $SYSTEM_CONTRACT_ADDR --abi $SYSTEM_CONTRACT_ABI \
-      getAddrRepository "${params}"  | jq -r .value0
+      getAddrRepository "${params}" | jq -r .value0
 }
 
 function list_branches {

diff --git a/v1_x/git-remote-gosh/Cargo.toml b/v1_x/git-remote-gosh/Cargo.toml
@@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
 rust-version = "1.68"
-version = '1.1.24'
+version = '1.1.25'
 
 [profile.profiling]
 debug = 1

diff --git a/v2_x/git-remote-gosh/Cargo.toml b/v2_x/git-remote-gosh/Cargo.toml
@@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
 rust-version = "1.68"
-version = '2.1.24'
+version = '2.1.25'
 
 [profile.profiling]
 debug = 1

diff --git a/v3_x/git-remote-gosh/Cargo.toml b/v3_x/git-remote-gosh/Cargo.toml
@@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
 rust-version = "1.68"
-version = '3.1.24'
+version = '3.1.25'
 
 [profile.profiling]
 debug = 1

diff --git a/v4_x/git-remote-gosh/Cargo.toml b/v4_x/git-remote-gosh/Cargo.toml
@@ -14,7 +14,7 @@ keywords = ['git-remote', 'git', 'gosh']
 name = 'git-remote-gosh'
 resolver = '2'
 rust-version = "1.68"
-version = '4.1.24'
+version = '4.1.25'
 
 [profile.profiling]
 debug = 1
@@ -104,12 +104,12 @@ version = '1.21.2'
 [dependencies.ton_client]
 git = 'https://github.com/tonlabs/ever-sdk.git'
 package = 'ton_client'
-tag = "1.41.1"
+tag = "1.43.1"
 
 [dependencies.ton_sdk]
 git = 'https://github.com/tonlabs/ever-sdk.git'
 package = 'ton_sdk'
-tag = "1.41.1"
+tag = "1.43.1"
 
 [dependencies.zstd]
 default-features = false

diff --git a/v4_x/git-remote-gosh/src/blockchain/call.rs b/v4_x/git-remote-gosh/src/blockchain/call.rs
@@ -166,7 +166,8 @@ impl BlockchainCall for Everscale {
             default_callback,
         )
         .instrument(info_span!("blockchain_client::send_message").or_current())
-        .await?;
+        .await
+            .map_err(|e| anyhow::format_err!("send_message error: {e}"))?;
 
         if let Some(expected_address) = expected_address {
             let start = Instant::now();

diff --git a/v4_x/git-remote-gosh/src/blockchain/commit/mod.rs b/v4_x/git-remote-gosh/src/blockchain/commit/mod.rs
@@ -107,7 +107,8 @@ pub async fn get_set_commit_created_at_time(
             },
         )
         .await
-        .map(|r| r.result)?;
+        .map(|r| r.result)
+            .map_err(|e| anyhow::format_err!("query error: {e}"))?;
 
         let extracted_messages = &result["data"]["blockchain"]["account"]["messages"];
         let messages: Messages = serde_json::from_value(extracted_messages.clone())?;

diff --git a/v4_x/git-remote-gosh/src/blockchain/commit/save.rs b/v4_x/git-remote-gosh/src/blockchain/commit/save.rs
@@ -285,7 +285,8 @@ pub async fn query_all_messages(
             },
         )
         .await
-        .map(|r| r.result)?;
+        .map(|r| r.result)
+            .map_err(|e| anyhow::format_err!("query error: {e}"))?;
 
         let nodes = &result["data"]["blockchain"]["account"]["messages"];
         let edges: Messages = serde_json::from_value(nodes.clone())?;
@@ -392,7 +393,8 @@ pub async fn is_transaction_ok(context: &Everscale, msg_id: &String) -> anyhow::
         },
     )
     .await
-    .map(|r| r.result)?;
+    .map(|r| r.result)
+        .map_err(|e| anyhow::format_err!("query error: {e}"))?;
 
     let trx: Vec<TrxInfo> = serde_json::from_value(result["data"]["transactions"].clone())?;
 

diff --git a/v4_x/git-remote-gosh/src/blockchain/contract/mod.rs b/v4_x/git-remote-gosh/src/blockchain/contract/mod.rs
@@ -149,6 +149,21 @@ impl GoshContract {
         Ok(serde_json::from_value::<T>(result)?)
     }
 
+    #[instrument(level = "info", skip_all)]
+    pub async fn run_local<T>(
+        &self,
+        context: &EverClient,
+        function_name: &str,
+        args: Option<serde_json::Value>,
+    ) -> anyhow::Result<T>
+        where
+            T: de::DeserializeOwned,
+    {
+        let result = run_local(context, self, function_name, args).await?;
+        tracing::trace!("run_local result: {:?}", result);
+        Ok(serde_json::from_value::<T>(result)?)
+    }
+
     pub async fn get_version(&self, context: &EverClient) -> anyhow::Result<String> {
         let result: GetVersionResult = self.read_state(context, "getVersion", None).await?;
         tracing::trace!("get_version result: {:?}", result);

diff --git a/v4_x/git-remote-gosh/src/blockchain/contract/wait_contracts_deployed.rs b/v4_x/git-remote-gosh/src/blockchain/contract/wait_contracts_deployed.rs
@@ -17,12 +17,12 @@ where
     B: BlockchainService + 'static,
 {
     tracing::trace!("wait_contracts_deployed: addresses={addresses:?}");
-    let mut deploymend_results: JoinSet<anyhow::Result<Vec<BlockchainContractAddress>>> =
+    let mut deployment_results: JoinSet<anyhow::Result<Vec<BlockchainContractAddress>>> =
         JoinSet::new();
     for chunk in addresses.chunks(MAX_ACCOUNTS_ADDRESSES_PER_QUERY) {
         let mut waiting_for_addresses = Vec::from(addresses);
         let b = blockchain.clone();
-        deploymend_results.spawn(
+        deployment_results.spawn(
             async move {
                 let mut iteration = 0;
                 while !waiting_for_addresses.is_empty() {
@@ -36,12 +36,10 @@ where
                         return Ok(waiting_for_addresses);
                     }
                     match b
-                        .get_contracts_state_raw_data(&waiting_for_addresses, true)
+                        .check_contracts_state(&waiting_for_addresses, true)
                         .await
                     {
-                        Ok(available_bocs) => {
-                            let found_addresses: Vec<BlockchainContractAddress> =
-                                available_bocs.into_keys().collect();
+                        Ok(found_addresses) => {
                             let available: HashSet<BlockchainContractAddress> =
                                 HashSet::from_iter(found_addresses.iter().cloned());
                             waiting_for_addresses.retain(|e| !available.contains(e));
@@ -66,11 +64,11 @@ where
                 } // While loop
                 Ok(vec![])
             } //move
-            .instrument(info_span!("tokio::spawn::wait_diff_deployed").or_current()),
+            .instrument(info_span!("tokio::spawn::wait_contracts_deployed").or_current()),
         );
     }
     let mut undeployed_contracts = HashSet::new();
-    while let Some(res) = deploymend_results.join_next().await {
+    while let Some(res) = deployment_results.join_next().await {
         let val = res??;
         for el in val {
             undeployed_contracts.insert(el);