[bug fix] Solved the problem that when the indexer parses txn info wi…

…thout transaction, it reports an error and exits, resulting in the block not being added to ES storage
starcoinorg · Jun 27, 2024 · 9ed9e7c · 9ed9e7c
1 parent 15fdd63
commit 9ed9e7c
Show file tree

Hide file tree

Showing 11 changed files with 103 additions and 48 deletions.
diff --git a/pom.xml b/pom.xml
@@ -31,7 +31,7 @@
         <dependency>
             <groupId>org.starcoin</groupId>
             <artifactId>sdk</artifactId>
-            <version>1.2.6</version>
+            <version>1.2.7</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>

diff --git a/starcoin-indexer/local.env b/starcoin-indexer/local.env
@@ -13,7 +13,7 @@ SWAP_API_URL=https://swap-api.starswap.xyz
 SWAP_CONTRACT_ADDR=0x8c109349c6bd91411d6bc962e080c4a3
 DS_URL=jdbc:postgresql://localhost/starcoin
 DB_SCHEMA=halley
-DB_USER_NAME=kaspa
-DB_PWD=kaspa
+DB_USER_NAME=starcoin
+DB_PWD=starcoin
 PROGRAM_ARGS=
 # auto_repair 9411700
diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/config/QuartzConfig.java
@@ -5,6 +5,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
 import org.springframework.scheduling.quartz.SchedulerFactoryBean;
 import org.starcoin.config.SearchJobFactory;
 import org.starcoin.indexer.handler.*;

diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/IndexerHandle.java
@@ -151,8 +151,11 @@ protected void executeInternal(JobExecutionContext jobExecutionContext) {
                     logger.info("rollback handle ok: {}", localBlockOffset);
                     return; //退出当前任务，重新添加从分叉点之后的block
                 }
+
                 //set event
-                ServiceUtils.addBlockToList(transactionRPCClient, blockList, block);
+                ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block);
+                blockList.add(block);
+
                 //update current header
                 currentHandleHeader = block.getHeader();
                 index++;

diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/RepairHandle.java
@@ -100,7 +100,8 @@ public boolean autoRepair(long startNumber, int count) {
             if (esBlock == null) {
                 logger.warn("es block not exist: {}", block.getHeader().getHeight());
                 try {
-                    ServiceUtils.addBlockToList(transactionRPCClient, blockList, block);
+                    ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block);
+                    blockList.add(block);
                 } catch (JSONRPC2SessionException e) {
                     logger.error("add block err:", e);
                 }
@@ -109,7 +110,8 @@ public boolean autoRepair(long startNumber, int count) {
             if (!block.getHeader().getBlockHash().equals(esBlock.getHeader().getBlockHash())) {
                 // fork block
                 try {
-                    ServiceUtils.addBlockToList(transactionRPCClient, blockList, block);
+                    ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block);
+                    blockList.add(block);
                 } catch (JSONRPC2SessionException e) {
                     logger.error("add fix block err:", e);
                 }
@@ -141,7 +143,8 @@ public void repair(long blockNumber) {
                 if (!blockOnChain.getHeader().getBlockHash().equals(blockOnEs.getHeader().getBlockHash())) {
 //                update block
                     List<Block> blockList = new ArrayList<>();
-                    ServiceUtils.addBlockToList(transactionRPCClient, blockList, blockOnChain);
+                    ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, blockOnChain);
+                    blockList.add(blockOnChain);
                     elasticSearchHandler.updateBlock(blockList);
                     logger.info("repair ok: {}", blockNumber);
                 } else {

diff --git a/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java b/starcoin-indexer/src/main/java/org/starcoin/indexer/handler/ServiceUtils.java
@@ -220,11 +220,19 @@ public static Map<String[], Long[]> getTokenReserveFromState(StateRPCClient stat
         return poolReserves;
     }
 
-    static void addBlockToList(TransactionRPCClient transactionRPCClient, List<Block> blockList, Block block) throws JSONRPC2SessionException {
-        List<Transaction> transactionList = transactionRPCClient.getBlockTransactions(block.getHeader().getBlockHash());
-        if (transactionList == null) {
+    public static void fetchTransactionsForBlock(TransactionRPCClient transactionRPCClient, Block block) throws JSONRPC2SessionException {
+        List<Transaction> transactionList;
+        try {
+            transactionList = transactionRPCClient.getBlockTransactions(block.getHeader().getBlockHash());
+        } catch (JSONRPC2SessionException e) {
+            logger.error("get block txn error:", e);
             return;
         }
+
+        if (transactionList == null || transactionList.isEmpty()) {
+            return;
+        }
+
         for (Transaction transaction : transactionList) {
             BlockMetadata metadata;
             Transaction userTransaction = transactionRPCClient.getTransactionByHash(transaction.getTransactionHash());
@@ -268,6 +276,6 @@ static void addBlockToList(TransactionRPCClient transactionRPCClient, List<Block
             }
         }
         block.setTransactionList(transactionList);
-        blockList.add(block);
+        // blockList.add(block);
     }
 }
diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/test/IndexerLogicBaseTest.java
@@ -1,12 +1,9 @@
 package org.starcoin.indexer.test;
 
-import org.junit.runner.RunWith;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.TestPropertySource;
-import org.springframework.test.context.junit4.SpringRunner;
 import org.starcoin.indexer.IndexerApplication;
 
-@RunWith(SpringRunner.class)
 @SpringBootTest(classes = IndexerApplication.class)
 @TestPropertySource(locations = "classpath:application-integrationtest.properties")
 public class IndexerLogicBaseTest {

diff --git a/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/ServiceUtilsTest.java b/starcoin-indexer/src/test/java/org/starcoin/indexer/utils/ServiceUtilsTest.java
@@ -0,0 +1,45 @@
+package org.starcoin.indexer.utils;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.starcoin.api.TransactionRPCClient;
+import org.starcoin.bean.Block;
+import org.starcoin.bean.BlockHeader;
+import org.starcoin.indexer.handler.ServiceUtils;
+import org.starcoin.indexer.test.IndexerLogicBaseTest;
+import org.starcoin.utils.ExceptionWrap;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class ServiceUtilsTest extends IndexerLogicBaseTest {
+
+    @Autowired
+    private TransactionRPCClient transactionRPCClient;
+
+    @Test
+    void testFetchTransactionsForBlock() {
+        List<String> blockHashes = Arrays.asList(
+                "0x6590d6769e9837e5fbc116daf53efa809f996e16cf55d4a713695094ac79aada",
+                "0x6b7ca9e9a1aa3f414caab285f214179108d5cf3fc2864efd9e7b5ed4461e7e80",
+                "0x042218a4750e696942fab2a875479a77f7c3c48049acb400a80c77b24aa06d36",
+                "0x721c15b57a22687dd90a439d85918f92f49937eef48c7c7afb72a7da43845589",
+                "0x9a68fd546aec8fdad955c0626cb9b20be121f932bb69f8a523f029248b125f51",
+                "0x73c0d5b604e130dedf0ecb77102e91de521bf38097b80d3e64c7ed58bc85c232",
+                "0x9161c9b51a7817cc2a18c190c99b40e96cb7cf4666ad63a7ffa33b2671c3aa31",
+                "0x020aadcabfbd57cc4c9059d16a91f8bc944b6c5c110d4fc72e6c346658035ecc"
+        );
+        List<Block> blocks = blockHashes.stream().map(ExceptionWrap.wrap(blockHash -> {
+            Block block = new Block();
+            BlockHeader header = new BlockHeader();
+            header.setBlockHash(blockHash);
+            block.setHeader(header);
+            ServiceUtils.fetchTransactionsForBlock(transactionRPCClient, block);
+            System.out.println("Block: " + blockHash);
+            return block;
+        })).collect(Collectors.toList());
+        blocks.forEach(System.out::println);
+
+    }
+}
diff --git a/starcoin-indexer/src/test/resources/application-integrationtest.properties b/starcoin-indexer/src/test/resources/application-integrationtest.properties
@@ -1,24 +1,27 @@
 # server.port=8300
-starcoin.seeds=${HOSTS}
-starcoin.network=${NETWORK}
-starcoin.bg_task.jobs=${BG_TASK_JOBS}
-starcoin.indexer.bulk_size=${BULK_SIZE}
-starcoin.indexer.txn_offset=${TXN_OFFSET}
+starcoin.seeds=vega.seed.starcoin.org
+starcoin.network=vega
+starcoin.bg_task.jobs=
+starcoin.indexer.bulk_size=100
+starcoin.indexer.txn_offset=0
 jasypt.encryptor.bean=encryptorBean
-jasypt.encryptor.password=${STARCOIN_ES_PWD}
-elasticsearch.host=${STARCOIN_ES_URL}
-elasticsearch.protocol=${STARCOIN_ES_PROTOCOL}
-elasticsearch.port=${STARCOIN_ES_PORT}
-elasticsearch.username=${STARCOIN_ES_USER}
+jasypt.encryptor.password=
+elasticsearch.host=localhost
+elasticsearch.protocol=http
+elasticsearch.port=9200
+elasticsearch.username=
 elasticsearch.connTimeout=10000
 elasticsearch.socketTimeout=10000
 elasticsearch.connectionRequestTimeout=2000
-swap.contract.address=${SWAP_CONTRACT_ADDR}
-starcoin.swap.api.url=${SWAP_API_URL}
+swap.contract.address=0x8c109349c6bd91411d6bc962e080c4a3
+starcoin.swap.api.url=https://swap-api.starswap.xyz
 logging.file.name=logs/indexer.log
+
+## Unittest config
+spring.profiles=unittest
 spring.datasource.hikari.minimum-idle=1
 spring.datasource.hikari.maximum-pool-size=2
-spring.datasource.url=${DS_URL}
-spring.datasource.username=${DB_USER_NAME}
-spring.datasource.password=${DB_PWD}
-spring.jpa.properties.hibernate.default_schema=${DB_SCHEMA}
+spring.datasource.url=jdbc:postgresql://localhost/starcoin
+spring.datasource.username=starcoin
+spring.datasource.password=starcoin
+spring.jpa.properties.hibernate.default_schema=vega
diff --git a/starcoin-indexer/src/test/resources/unit-test.env b/starcoin-indexer/src/test/resources/unit-test.env
diff --git a/starcoin-scan-api/local.env b/starcoin-scan-api/local.env
@@ -0,0 +1,13 @@
+STARCOIN_ES_URL=localhost
+STARCOIN_ES_PROTOCOL=http
+STARCOIN_ES_PORT=9200
+STARCOIN_ES_USER=
+STARCOIN_ES_INDEX_VERSION=
+STARCOIN_ES_PWD=
+MAIN_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=main
+BARNARD_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=barnard
+HALLEY_DS_URL=jdbc:postgresql://localhost/starcoin?currentSchema=halley
+DS_URL=jdbc:postgresql://localhost/starcoin
+STARCOIN_USER_DS_URL="jdbc:postgresql://localhost/starcoin?currentSchema=starcoin_user"
+DB_USER_NAME=starcoin
+DB_PWD=starcoin