Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup record access #2792

Merged
merged 4 commits into from
Jan 25, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ public Set<String> allPropertyNames() {
"ledger.transfers.maxLen",
"ledger.tokenTransfers.maxLen",
"ledger.nftTransfers.maxLen",
"ledger.records.maxQueryableByAccount",
"ledger.schedule.txExpiryTimeSecs",
"rates.intradayChangeLimitPercent",
"rates.midnightCheckInterval",
Expand Down Expand Up @@ -383,6 +384,7 @@ public static Function<String, Object> transformFor(String prop) {
entry("ledger.nftTransfers.maxLen", AS_INT),
entry("ledger.totalTinyBarFloat", AS_LONG),
entry("ledger.schedule.txExpiryTimeSecs", AS_INT),
entry("ledger.records.maxQueryableByAccount", AS_INT),
entry("iss.dumpFcms", AS_BOOLEAN),
entry("iss.resetPeriod", AS_INT),
entry("iss.roundsToDump", AS_INT),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ public class GlobalDynamicProperties {
private boolean expandSigsFromLastSignedState;
private long maxAggregateContractKvPairs;
private int maxIndividualContractKvPairs;
private int maxMostRecentQueryableRecords;

@Inject
public GlobalDynamicProperties(
Expand Down Expand Up @@ -174,6 +175,7 @@ public void reload() {
expandSigsFromLastSignedState = properties.getBooleanProperty("sigs.expandFromLastSignedState");
maxAggregateContractKvPairs = properties.getLongProperty("contracts.maxKvPairs.aggregate");
maxIndividualContractKvPairs = properties.getIntProperty("contracts.maxKvPairs.individual");
maxMostRecentQueryableRecords = properties.getIntProperty("ledger.records.maxQueryableByAccount");
}

public int maxTokensPerAccount() {
Expand Down Expand Up @@ -403,4 +405,8 @@ public long maxAggregateContractKvPairs() {
public int maxIndividualContractKvPairs() {
return maxIndividualContractKvPairs;
}

public int maxNumQueryableRecords() {
return maxMostRecentQueryableRecords;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@
import com.hederahashgraph.api.proto.java.ResponseType;
import com.hederahashgraph.fee.CryptoFeeBuilder;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.Map;

import static com.hedera.services.queries.meta.GetTxnRecordAnswer.PAYER_RECORDS_CTX_KEY;
import static com.hedera.services.utils.EntityNum.fromAccountId;
import static com.hedera.services.utils.MiscUtils.putIfNotNull;

@Singleton
public final class GetAccountRecordsResourceUsage implements QueryResourceUsageEstimator {
Expand All @@ -54,12 +57,21 @@ public boolean applicableTo(final Query query) {
}

@Override
public FeeData usageGiven(final Query query, final StateView view, final Map<String, Object> ignoreCtx) {
return usageGivenType(query, view, query.getCryptoGetAccountRecords().getHeader().getResponseType());
public FeeData usageGiven(final Query query, final StateView view, final Map<String, Object> queryCtx) {
return usageFor(query, view, query.getCryptoGetAccountRecords().getHeader().getResponseType(), queryCtx);
}

@Override
public FeeData usageGivenType(final Query query, final StateView view, final ResponseType type) {
return usageFor(query, view, type, null);
}

private FeeData usageFor(
final Query query,
final StateView view,
final ResponseType stateProofType,
@Nullable final Map<String, Object> queryCtx
) {
final var op = query.getCryptoGetAccountRecords();
final var target = fromAccountId(op.getAccountID());
if (!view.accounts().containsKey(target)) {
Expand All @@ -69,7 +81,8 @@ public FeeData usageGivenType(final Query query, final StateView view, final Res
* status code); so just return the default {@code FeeData} here. */
return FeeData.getDefaultInstance();
}
final var records = answerFunctions.accountRecords(view, op);
return usageEstimator.getCryptoAccountRecordsQueryFeeMatrices(records, type);
final var records = answerFunctions.mostRecentRecords(view, op);
putIfNotNull(queryCtx, PAYER_RECORDS_CTX_KEY, records);
return usageEstimator.getCryptoAccountRecordsQueryFeeMatrices(records, stateProofType);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -71,24 +71,21 @@ public boolean applicableTo(final Query query) {

@Override
public FeeData usageGiven(final Query query, final StateView view, @Nullable final Map<String, Object> queryCtx) {
return usageFor(query, view, query.getTransactionGetRecord().getHeader().getResponseType(), queryCtx);
return usageFor(query, query.getTransactionGetRecord().getHeader().getResponseType(), queryCtx);
}

@Override
public FeeData usageGivenType(final Query query, final StateView view, final ResponseType type) {
return usageFor(query, view, type, null);
return usageFor(query, type, null);
}

private FeeData usageFor(
final Query query,
final StateView view,
final ResponseType stateProofType,
@Nullable final Map<String, Object> queryCtx
) {
final var op = query.getTransactionGetRecord();
final var txnRecord = answerFunctions
.txnRecord(recordCache, view, op)
.orElse(MISSING_RECORD_STANDIN);
final var txnRecord = answerFunctions.txnRecord(recordCache, op).orElse(MISSING_RECORD_STANDIN);
var usages = usageEstimator.getTransactionRecordQueryFeeMatrices(txnRecord, stateProofType);
if (txnRecord != MISSING_RECORD_STANDIN) {
putIfNotNull(queryCtx, PRIORITY_RECORD_CTX_KEY, txnRecord);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,54 +21,110 @@
*/

import com.hedera.services.context.primitives.StateView;
import com.hedera.services.context.properties.GlobalDynamicProperties;
import com.hedera.services.records.RecordCache;
import com.hedera.services.state.merkle.MerkleAccount;
import com.hedera.services.state.submerkle.ExpirableTxnRecord;
import com.hedera.services.state.submerkle.TxnId;
import com.hedera.services.utils.EntityNum;
import com.hederahashgraph.api.proto.java.CryptoGetAccountRecordsQuery;
import com.hederahashgraph.api.proto.java.TransactionGetRecordQuery;
import com.hederahashgraph.api.proto.java.TransactionRecord;

import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.ArrayList;
import java.util.Collections;
import java.util.ConcurrentModificationException;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Optional;

@Singleton
public class AnswerFunctions {
private final GlobalDynamicProperties dynamicProperties;

@Inject
public AnswerFunctions() {
// Default Constructor
public AnswerFunctions(GlobalDynamicProperties dynamicProperties) {
this.dynamicProperties = dynamicProperties;
}

public List<TransactionRecord> accountRecords(final StateView view, final CryptoGetAccountRecordsQuery op) {
final var key = EntityNum.fromAccountId(op.getAccountID());
final var account = view.accounts().get(key);
return ExpirableTxnRecord.allToGrpc(account.recordList());
/**
* Returns the most recent payer records available for an account in the given {@link StateView}.
*
* Note that at <b>most</b> {@link GlobalDynamicProperties#maxNumQueryableRecords()} records will be available,
* even if the given account has paid for more than this number of transactions in the last 180 seconds.
*
* @param view the view of the world state to get payer records from
* @param op the query with the target payer account
* @return the most recent available records for the given payer
*/
public List<TransactionRecord> mostRecentRecords(final StateView view, final CryptoGetAccountRecordsQuery op) {
final var targetId = EntityNum.fromAccountId(op.getAccountID());
final var targetAccount = view.accounts().get(targetId);
if (targetAccount == null) {
return Collections.emptyList();
}
final var numAvailable = targetAccount.numRecords();
final var maxQueryable = dynamicProperties.maxNumQueryableRecords();
return numAvailable <= maxQueryable
? recordsFrom(targetAccount, numAvailable)
: mostRecentFrom(targetAccount, maxQueryable, numAvailable);
}

public Optional<TransactionRecord> txnRecord(
final RecordCache recordCache,
final StateView view,
final TransactionGetRecordQuery query
) {
final var txnId = query.getTransactionID();
/**
* Returns the record of the requested transaction from the given {@link RecordCache}, if available.
*
* @param recordCache the cache to get the record from
* @param op the query with the target transaction id
* @return the transaction record if available
*/
public Optional<TransactionRecord> txnRecord(final RecordCache recordCache, final TransactionGetRecordQuery op) {
final var txnId = op.getTransactionID();
final var expirableTxnRecord = recordCache.getPriorityRecord(txnId);
if (expirableTxnRecord != null) {
return Optional.of(expirableTxnRecord.asGrpc());
} else {
try {
final var id = txnId.getAccountID();
final var account = view.accounts().get(EntityNum.fromAccountId(id));
final var searchableId = TxnId.fromGrpc(txnId);
return account.recordList()
.stream()
.filter(r -> r.getTxnId().equals(searchableId))
.findAny()
.map(ExpirableTxnRecord::asGrpc);
} catch (final Exception ignore) {
return Optional.empty();
return Optional.ofNullable(expirableTxnRecord).map(ExpirableTxnRecord::asGrpc);
}

/* --- Internal helpers --- */
/**
* Returns up to {@code n} payer records from an account in gRPC form.
*
* @param account the account of interest
* @param n the expected number of records in the account
* @return the available records
*/
private List<TransactionRecord> recordsFrom(final MerkleAccount account, final int n) {
return mostRecentFrom(account, n, n);
}

/**
* Returns up to the last {@code m}-of-{@code n} payer records from an account in gRPC form.
*
* Since records are added FIFO to the payer account, the last records are the most recent;
* and presumably the most interesting.
*
* If the given {@link MerkleAccount} is from the working state (as in release 0.22), then
* this method acts on a best-effort basis, and returns only the relevant records it could
* iterate over before hitting a {@link ConcurrentModificationException} or {@link NoSuchElementException}.
*
* @param account the account of interest
* @param m the maximum number of records to return
* @param n the expected number of records in the account
* @return the available records
*/
private List<TransactionRecord> mostRecentFrom(final MerkleAccount account, final int m, final int n) {
final List<TransactionRecord> ans = new ArrayList<>();
final Iterator<ExpirableTxnRecord> iter = account.recordIterator();
try {
for (int i = 0, cutoff = n - m; i < n; i++) {
final var nextRecord = iter.next();
if (i >= cutoff) {
ans.add(nextRecord.asGrpc());
}
}
rbair23 marked this conversation as resolved.
Show resolved Hide resolved
} catch (ConcurrentModificationException | NoSuchElementException ignore) {
/* Records expired while we were iterating the list, return only what we could find. */
}
return ans;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,22 @@
import com.hedera.services.queries.answering.AnswerFunctions;
import com.hedera.services.txns.validation.OptionValidator;
import com.hedera.services.utils.SignedTxnAccessor;
import com.hederahashgraph.api.proto.java.CryptoGetAccountRecordsQuery;
import com.hederahashgraph.api.proto.java.CryptoGetAccountRecordsResponse;
import com.hederahashgraph.api.proto.java.HederaFunctionality;
import com.hederahashgraph.api.proto.java.Query;
import com.hederahashgraph.api.proto.java.Response;
import com.hederahashgraph.api.proto.java.ResponseCodeEnum;
import com.hederahashgraph.api.proto.java.TransactionRecord;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Singleton;
import java.util.List;
import java.util.Map;
import java.util.Optional;

import static com.hedera.services.queries.meta.GetTxnRecordAnswer.PAYER_RECORDS_CTX_KEY;
import static com.hederahashgraph.api.proto.java.HederaFunctionality.CryptoGetAccountRecords;
import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.OK;
import static com.hederahashgraph.api.proto.java.ResponseType.COST_ANSWER;
Expand Down Expand Up @@ -69,6 +75,27 @@ public Response responseGiven(
final StateView view,
final ResponseCodeEnum validity,
final long cost
) {
return responseFor(query, view, validity, cost, null);
}

@Override
public Response responseGiven(
final Query query,
final StateView view,
final ResponseCodeEnum validity,
final long cost,
final Map<String, Object> queryCtx
) {
return responseFor(query, view, validity, cost, queryCtx);
}

private Response responseFor(
final Query query,
final StateView view,
final ResponseCodeEnum validity,
final long cost,
final @Nullable Map<String, Object> queryCtx
) {
final var op = query.getCryptoGetAccountRecords();
final var response = CryptoGetAccountRecordsResponse.newBuilder();
Expand All @@ -81,9 +108,7 @@ public Response responseGiven(
response.setAccountID(op.getAccountID());
response.setHeader(costAnswerHeader(OK, cost));
} else {
response.setHeader(answerOnlyHeader(OK));
response.setAccountID(op.getAccountID());
response.addAllRecords(answerFunctions.accountRecords(view, op));
setAnswerOnly(response, view, op, queryCtx);
}
}

Expand All @@ -92,6 +117,22 @@ public Response responseGiven(
.build();
}

@SuppressWarnings("unchecked")
private void setAnswerOnly(
final CryptoGetAccountRecordsResponse.Builder response,
final StateView view,
final CryptoGetAccountRecordsQuery op,
final @Nullable Map<String, Object> queryCtx
) {
response.setHeader(answerOnlyHeader(OK));
response.setAccountID(op.getAccountID());
if (queryCtx != null && queryCtx.containsKey(PAYER_RECORDS_CTX_KEY)) {
response.addAllRecords((List<TransactionRecord>) queryCtx.get(PAYER_RECORDS_CTX_KEY));
} else {
response.addAllRecords(answerFunctions.mostRecentRecords(view, op));
}
}

@Override
public ResponseCodeEnum checkValidity(final Query query, final StateView view) {
final var id = query.getCryptoGetAccountRecords().getAccountID();
Expand All @@ -112,6 +153,6 @@ public HederaFunctionality canonicalFunction() {
@Override
public Optional<SignedTxnAccessor> extractPaymentFrom(final Query query) {
final var paymentTxn = query.getCryptoGetAccountRecords().getHeader().getPayment();
return Optional.ofNullable(SignedTxnAccessor.uncheckedFrom(paymentTxn));
return Optional.of(SignedTxnAccessor.uncheckedFrom(paymentTxn));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What was the significance of this change?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The gRPC accessors never return null, so IntelliJ was marking this Optional.ofNullable() as a code smell.

}
}
Loading