From e55c38037ef0afd9761eba404bb47f17184c9de1 Mon Sep 17 00:00:00 2001 From: Chengxuan Xing Date: Tue, 24 Sep 2024 13:02:37 +0100 Subject: [PATCH 1/3] adding locks for witness gen Signed-off-by: Chengxuan Xing --- solidity/package.json | 3 +- ...to_anon_enc_nullifier_kyc_cost_analysis.ts | 107 +++++++++++------- solidity/test/lib/utils.ts | 2 +- 3 files changed, 71 insertions(+), 41 deletions(-) diff --git a/solidity/package.json b/solidity/package.json index 47696e9..50ee394 100644 --- a/solidity/package.json +++ b/solidity/package.json @@ -24,6 +24,7 @@ "@iden3/contracts": "^2.0.5", "@openzeppelin/contracts": "^5.0.1", "@openzeppelin/contracts-upgradeable": "^5.0.2", - "@openzeppelin/hardhat-upgrades": "^3.2.1" + "@openzeppelin/hardhat-upgrades": "^3.2.1", + "async-lock": "^1.4.1" } } diff --git a/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts b/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts index a8ffced..0f112c7 100644 --- a/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts +++ b/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts @@ -21,6 +21,8 @@ import { expect } from 'chai'; import { loadCircuit, encodeProof, newEncryptionNonce, kycHash } from 'zeto-js'; import { groth16 } from 'snarkjs'; import { stringifyBigInts } from 'maci-crypto'; +import AsyncLock from 'async-lock'; +const lock = new AsyncLock(); import { Merkletree, InMemoryDB, str2Bytes } from '@iden3/js-merkletree'; import { UTXO, @@ -42,10 +44,20 @@ import { } from '../utils'; import { deployZeto } from '../lib/deploy'; -const TOTAL_AMOUNT = parseInt(process.env.TOTAL_ROUNDS || '1000'); +const TOTAL_AMOUNT = parseInt(process.env.TOTAL_ROUNDS || '10'); const TX_CONCURRENCY = parseInt(process.env.TX_CONCURRENCY || '30'); -describe.skip('(Gas cost analysis) Zeto based fungible token with anonymity using nullifiers and encryption with KYC', function () { +export interface PreparedTransferData { + signer: User; + nullifiers: [BigNumberish, BigNumberish]; + outputCommitments: [BigNumberish, BigNumberish]; + utxosRoot: BigNumberish; + encryptedValues: BigNumberish[]; + encryptionNonce: BigNumberish; + encodedProof: any; +} + +describe.only('(Gas cost analysis) Zeto based fungible token with anonymity using nullifiers and encryption with KYC', function () { let deployer: Signer; let Alice: User; let Bob: User; @@ -244,7 +256,7 @@ describe.skip('(Gas cost analysis) Zeto based fungible token with anonymity usin }).timeout(6000000000000); it(`Alice transfer ${TOTAL_AMOUNT} tokens to Bob in ${atLeastHalfAmount} txs`, async function () { - const totalTxs = unspentAliceUTXOs.length / 2; + const totalTxs = Math.floor(unspentAliceUTXOs.length / 2); const utxosRoot = await smtAlice.root(); // get the root before all transfer and use it for all the proofs let promises = []; // Alice generates inclusion proofs for the identities in the transaction @@ -314,7 +326,7 @@ describe.skip('(Gas cost analysis) Zeto based fungible token with anonymity usin ); // If we reach the concurrency limit, wait for the current batch to finish - if (promises.length >= 1) { + if (promises.length >= TX_CONCURRENCY) { await Promise.all(promises); promises = []; // Reset promises for the next batch } @@ -334,43 +346,56 @@ describe.skip('(Gas cost analysis) Zeto based fungible token with anonymity usin const startingBalance = await erc20.balanceOf(Bob.ethAddress); const root = await smtBob.root(); + let promises = []; for (let i = 0; i < unspentBobUTXOs.length; i++) { if (unspentBobUTXOs[i].value) { - const utxoToWithdraw = unspentBobUTXOs[i]; - const nullifier1 = newNullifier(utxoToWithdraw, Bob); - - const proof1 = await smtBob.generateCircomVerifierProof( - utxoToWithdraw.hash, - root - ); - const proof2 = await smtBob.generateCircomVerifierProof(0n, root); - const merkleProofs = [ - proof1.siblings.map((s) => s.bigInt()), - proof2.siblings.map((s) => s.bigInt()), - ]; - - const { nullifiers, outputCommitments, encodedProof } = - await prepareNullifierWithdrawProof( - Bob, - [utxoToWithdraw, ZERO_UTXO], - [nullifier1, ZERO_UTXO], - newUTXO(0, Bob), - root.bigInt(), - merkleProofs - ); - - // Bob withdraws UTXOs to ERC20 tokens - await doWithdraw( - zeto, - Bob.signer, - 1, - nullifiers, - outputCommitments[0], - root.bigInt(), - encodedProof, - withdrawGasCostHistory + promises.push( + (async () => { + const utxoToWithdraw = unspentBobUTXOs[i]; + const nullifier1 = newNullifier(utxoToWithdraw, Bob); + + const proof1 = await smtBob.generateCircomVerifierProof( + utxoToWithdraw.hash, + root + ); + const proof2 = await smtBob.generateCircomVerifierProof(0n, root); + const merkleProofs = [ + proof1.siblings.map((s) => s.bigInt()), + proof2.siblings.map((s) => s.bigInt()), + ]; + const { nullifiers, outputCommitments, encodedProof } = + await prepareNullifierWithdrawProof( + Bob, + [utxoToWithdraw, ZERO_UTXO], + [nullifier1, ZERO_UTXO], + newUTXO(0, Bob), + root.bigInt(), + merkleProofs + ); + + // Bob withdraws UTXOs to ERC20 tokens + await doWithdraw( + zeto, + Bob.signer, + 1, + nullifiers, + outputCommitments[0], + root.bigInt(), + encodedProof, + withdrawGasCostHistory + ); + })() ); } + // If we reach the concurrency limit, wait for the current batch to finish + if (promises.length >= TX_CONCURRENCY) { + await Promise.all(promises); + promises = []; // Reset promises for the next batch + } + } + // Run any remaining promises that didn’t fill the batch + if (promises.length > 0) { + await Promise.all(promises); } writeGasCostsToCSV( `${reportPrefix}withdraw_gas_costs.csv`, @@ -490,7 +515,10 @@ describe.skip('(Gas cost analysis) Zeto based fungible token with anonymity usin outputOwnerPublicKeys, ...encryptInputs, }; - const witness = await circuit.calculateWTNSBin(inputObj, true); + const witness = await lock.acquire('proofGen', async () => { + // this lock is added for https://github.com/hyperledger-labs/zeto/issues/80, which only happens for Transfer circuit, not deposit/mint + return circuit.calculateWTNSBin(inputObj, true); + }); const timeWithnessCalculation = Date.now() - startWitnessCalculation; const startProofGeneration = Date.now(); @@ -533,7 +561,8 @@ describe.skip('(Gas cost analysis) Zeto based fungible token with anonymity usin root, encryptionNonce, encryptedValues, - encodedProof + encodedProof, + '0x' ); const result: ContractTransactionReceipt | null = await tx.wait(); console.log( diff --git a/solidity/test/lib/utils.ts b/solidity/test/lib/utils.ts index 9f0c42c..126524a 100644 --- a/solidity/test/lib/utils.ts +++ b/solidity/test/lib/utils.ts @@ -89,7 +89,7 @@ export async function doMint(zetoTokenContract: any, minter: Signer, outputs: UT } export async function doDeposit(zetoTokenContract: any, depositUser: Signer, amount:any, commitment: any, proof: any, gasHistories?:number[]): Promise { - const tx = await zetoTokenContract.connect(depositUser).deposit(amount, commitment, proof); + const tx = await zetoTokenContract.connect(depositUser).deposit(amount, commitment, proof, "0x"); const result = await tx.wait(); console.log(`Method deposit() complete. Gas used: ${result?.gasUsed}`); if (result?.gasUsed && Array.isArray(gasHistories)) { From 27b43eb60f6ec42b314b9e7e0b0ae215b2814d3a Mon Sep 17 00:00:00 2001 From: Chengxuan Xing Date: Tue, 24 Sep 2024 13:02:59 +0100 Subject: [PATCH 2/3] revert changes Signed-off-by: Chengxuan Xing --- .../gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts b/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts index 0f112c7..46c7817 100644 --- a/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts +++ b/solidity/test/gas_cost/zeto_anon_enc_nullifier_kyc_cost_analysis.ts @@ -44,7 +44,7 @@ import { } from '../utils'; import { deployZeto } from '../lib/deploy'; -const TOTAL_AMOUNT = parseInt(process.env.TOTAL_ROUNDS || '10'); +const TOTAL_AMOUNT = parseInt(process.env.TOTAL_ROUNDS || '1000'); const TX_CONCURRENCY = parseInt(process.env.TX_CONCURRENCY || '30'); export interface PreparedTransferData { @@ -57,7 +57,7 @@ export interface PreparedTransferData { encodedProof: any; } -describe.only('(Gas cost analysis) Zeto based fungible token with anonymity using nullifiers and encryption with KYC', function () { +describe.skip('(Gas cost analysis) Zeto based fungible token with anonymity using nullifiers and encryption with KYC', function () { let deployer: Signer; let Alice: User; let Bob: User; From 2766782f83838eef2321d15582365b4e3ad29fd8 Mon Sep 17 00:00:00 2001 From: Jim Zhang Date: Tue, 24 Sep 2024 09:31:23 -0400 Subject: [PATCH 3/3] Move ErrNotFound from internal to pkg Signed-off-by: Jim Zhang --- go-sdk/internal/sparse-merkle-tree/smt/merkletree.go | 3 +-- go-sdk/internal/sparse-merkle-tree/smt/merkletree_test.go | 3 +-- go-sdk/internal/sparse-merkle-tree/storage/sql.go | 4 ++-- .../storage => pkg/sparse-merkle-tree/core}/errors.go | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) rename go-sdk/{internal/sparse-merkle-tree/storage => pkg/sparse-merkle-tree/core}/errors.go (97%) diff --git a/go-sdk/internal/sparse-merkle-tree/smt/merkletree.go b/go-sdk/internal/sparse-merkle-tree/smt/merkletree.go index 3a2dfcc..ca490e9 100644 --- a/go-sdk/internal/sparse-merkle-tree/smt/merkletree.go +++ b/go-sdk/internal/sparse-merkle-tree/smt/merkletree.go @@ -23,7 +23,6 @@ import ( "github.com/hyperledger-labs/zeto/go-sdk/internal/log" "github.com/hyperledger-labs/zeto/go-sdk/internal/sparse-merkle-tree/node" - "github.com/hyperledger-labs/zeto/go-sdk/internal/sparse-merkle-tree/storage" "github.com/hyperledger-labs/zeto/go-sdk/internal/sparse-merkle-tree/utils" "github.com/hyperledger-labs/zeto/go-sdk/pkg/sparse-merkle-tree/core" ) @@ -46,7 +45,7 @@ func NewMerkleTree(db core.Storage, maxLevels int) (core.SparseMerkleTree, error mt := sparseMerkleTree{db: db, maxLevels: maxLevels} root, err := mt.db.GetRootNodeIndex() - if err == storage.ErrNotFound { + if err == core.ErrNotFound { mt.rootKey = node.ZERO_INDEX err = mt.db.UpsertRootNodeIndex(mt.rootKey) if err != nil { diff --git a/go-sdk/internal/sparse-merkle-tree/smt/merkletree_test.go b/go-sdk/internal/sparse-merkle-tree/smt/merkletree_test.go index d80ff83..77ef41c 100644 --- a/go-sdk/internal/sparse-merkle-tree/smt/merkletree_test.go +++ b/go-sdk/internal/sparse-merkle-tree/smt/merkletree_test.go @@ -20,7 +20,6 @@ import ( "fmt" "testing" - "github.com/hyperledger-labs/zeto/go-sdk/internal/sparse-merkle-tree/storage" "github.com/hyperledger-labs/zeto/go-sdk/pkg/sparse-merkle-tree/core" "github.com/stretchr/testify/assert" ) @@ -33,7 +32,7 @@ func (ms *mockStorage) GetRootNodeIndex() (core.NodeIndex, error) { if ms.GetRootNodeIndex_customError { return nil, fmt.Errorf("nasty error in get root") } - return nil, storage.ErrNotFound + return nil, core.ErrNotFound } func (ms *mockStorage) UpsertRootNodeIndex(core.NodeIndex) error { return fmt.Errorf("nasty error in upsert root") diff --git a/go-sdk/internal/sparse-merkle-tree/storage/sql.go b/go-sdk/internal/sparse-merkle-tree/storage/sql.go index 6c08182..73418f6 100644 --- a/go-sdk/internal/sparse-merkle-tree/storage/sql.go +++ b/go-sdk/internal/sparse-merkle-tree/storage/sql.go @@ -47,7 +47,7 @@ func (s *sqlStorage) GetRootNodeIndex() (core.NodeIndex, error) { } err := s.p.DB().Table(core.TreeRootsTable).First(&root).Error if err == gorm.ErrRecordNotFound { - return nil, ErrNotFound + return nil, core.ErrNotFound } else if err != nil { return nil, err } @@ -121,7 +121,7 @@ func getNode(batchOrDb *gorm.DB, nodesTableName string, ref core.NodeIndex) (cor } err := batchOrDb.Table(nodesTableName).First(&n).Error if err == gorm.ErrRecordNotFound { - return nil, ErrNotFound + return nil, core.ErrNotFound } else if err != nil { return nil, err } diff --git a/go-sdk/internal/sparse-merkle-tree/storage/errors.go b/go-sdk/pkg/sparse-merkle-tree/core/errors.go similarity index 97% rename from go-sdk/internal/sparse-merkle-tree/storage/errors.go rename to go-sdk/pkg/sparse-merkle-tree/core/errors.go index 379ed77..6e36ede 100644 --- a/go-sdk/internal/sparse-merkle-tree/storage/errors.go +++ b/go-sdk/pkg/sparse-merkle-tree/core/errors.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package core import "errors"