Skip to content

Commit

Permalink
issue-1146: base cache capacity calculations relative to the MaxNodes…
Browse files Browse the repository at this point in the history 
… value, add cache capacity sensors (#2534)
  • Loading branch information
@debnatkh
debnatkh authored Nov 21, 2024
1 parent 26e6337 commit d47444e
Show file tree
Hide file tree
Showing 9 changed files with 103 additions and 6 deletions.
10 changes: 7 additions & 3 deletions cloud/filestore/config/storage.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,11 +360,15 @@ message TStorageConfig
optional bool InMemoryIndexCacheEnabled = 371;
// Capacity of in-memory index cache, in number of entries per each table
optional uint64 InMemoryIndexCacheNodesCapacity = 372;
reserved 373; // InMemoryIndexCacheNodesVerCapacity
// Ratio of maximal number of nodes in the fs to the in-memory index cache
// capacity. I. e., if there are max 1000 nodes and ratio is set to 10,
// then there can be max 100 entries in cache. If both capacity and ratio
// are set, the greater value is used.
optional uint64 InMemoryIndexCacheNodesToNodesCapacityRatio = 373;
optional uint64 InMemoryIndexCacheNodeAttrsCapacity = 374;
reserved 375; // InMemoryIndexCacheNodeAttrsVerCapacity
optional uint64 InMemoryIndexCacheNodesToNodeAttrsCapacityRatio = 375;
optional uint64 InMemoryIndexCacheNodeRefsCapacity = 376;
reserved 377; // InMemoryIndexCacheNodeRefsVerCapacity
optional uint64 InMemoryIndexCacheNodesToNodeRefsCapacityRatio = 377;

// Used to send non-network metrics as network ones to HIVE,
// while we use them for load balancing
Expand Down
3 changes: 3 additions & 0 deletions cloud/filestore/libs/storage/core/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,8 +206,11 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
\
xxx(InMemoryIndexCacheEnabled, bool, false )\
xxx(InMemoryIndexCacheNodesCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodesToNodesCapacityRatio, ui64, 0 )\
xxx(InMemoryIndexCacheNodeAttrsCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodesToNodeAttrsCapacityRatio,ui64, 0 )\
xxx(InMemoryIndexCacheNodeRefsCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodesToNodeRefsCapacityRatio, ui64, 0 )\
xxx(InMemoryIndexCacheLoadOnTabletStart, bool, false )\
xxx(InMemoryIndexCacheLoadOnTabletStartRowsPerTx, ui64, 1000000 )\
\
Expand Down
3 changes: 3 additions & 0 deletions cloud/filestore/libs/storage/core/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,11 @@ class TStorageConfig

bool GetInMemoryIndexCacheEnabled() const;
ui64 GetInMemoryIndexCacheNodesCapacity() const;
ui64 GetInMemoryIndexCacheNodesToNodesCapacityRatio() const;
ui64 GetInMemoryIndexCacheNodeAttrsCapacity() const;
ui64 GetInMemoryIndexCacheNodesToNodeAttrsCapacityRatio() const;
ui64 GetInMemoryIndexCacheNodeRefsCapacity() const;
ui64 GetInMemoryIndexCacheNodesToNodeRefsCapacityRatio() const;
bool GetInMemoryIndexCacheLoadOnTabletStart() const;
ui64 GetInMemoryIndexCacheLoadOnTabletStartRowsPerTx() const;

Expand Down
3 changes: 3 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_actor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,11 @@ class TIndexTabletActor final
std::atomic<i64> InMemoryIndexStateRWCount{0};

std::atomic<i64> InMemoryIndexStateNodesCount;
std::atomic<i64> InMemoryIndexStateNodesCapacity;
std::atomic<i64> InMemoryIndexStateNodeRefsCount;
std::atomic<i64> InMemoryIndexStateNodeRefsCapacity;
std::atomic<i64> InMemoryIndexStateNodeAttrsCount;
std::atomic<i64> InMemoryIndexStateNodeAttrsCapacity;
std::atomic<i64> InMemoryIndexStateIsExhaustive;

// Data stats
Expand Down
12 changes: 12 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -291,12 +291,21 @@ void TIndexTabletActor::TMetrics::Register(
REGISTER_AGGREGATABLE_SUM(
InMemoryIndexStateNodesCount,
EMetricType::MT_ABSOLUTE);
REGISTER_AGGREGATABLE_SUM(
InMemoryIndexStateNodesCapacity,
EMetricType::MT_ABSOLUTE);
REGISTER_AGGREGATABLE_SUM(
InMemoryIndexStateNodeRefsCount,
EMetricType::MT_ABSOLUTE);
REGISTER_AGGREGATABLE_SUM(
InMemoryIndexStateNodeRefsCapacity,
EMetricType::MT_ABSOLUTE);
REGISTER_AGGREGATABLE_SUM(
InMemoryIndexStateNodeAttrsCount,
EMetricType::MT_ABSOLUTE);
REGISTER_AGGREGATABLE_SUM(
InMemoryIndexStateNodeAttrsCapacity,
EMetricType::MT_ABSOLUTE);
REGISTER_AGGREGATABLE_SUM(
InMemoryIndexStateIsExhaustive,
EMetricType::MT_ABSOLUTE);
Expand Down Expand Up @@ -487,8 +496,11 @@ void TIndexTabletActor::TMetrics::Update(
Store(NodeIndexCacheNodeCount, nodeIndexCacheStats.NodeCount);

Store(InMemoryIndexStateNodesCount, inMemoryIndexStateStats.NodesCount);
Store(InMemoryIndexStateNodesCapacity, inMemoryIndexStateStats.NodesCapacity);
Store(InMemoryIndexStateNodeRefsCount, inMemoryIndexStateStats.NodeRefsCount);
Store(InMemoryIndexStateNodeRefsCapacity, inMemoryIndexStateStats.NodeRefsCapacity);
Store(InMemoryIndexStateNodeAttrsCount, inMemoryIndexStateStats.NodeAttrsCount);
Store(InMemoryIndexStateNodeAttrsCapacity, inMemoryIndexStateStats.NodeAttrsCapacity);
Store(InMemoryIndexStateIsExhaustive, inMemoryIndexStateStats.IsNodeRefsExhaustive);

Store(
Expand Down
26 changes: 23 additions & 3 deletions cloud/filestore/libs/storage/tablet/tablet_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,17 @@ bool IsValid(const NProto::TFileStorePerformanceProfile& profile)
&& profile.GetDefaultPostponedRequestWeight();
}

ui64 CalculateInMemoryIndexCacheCapacity(
const ui64 capacity,
const ui64 maxNodes,
const ui64 nodesToCapacityRatio)
{
if (nodesToCapacityRatio == 0) {
return capacity;
}
return Max(capacity, maxNodes / nodesToCapacityRatio);
}

} // namespace

////////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -126,9 +137,18 @@ void TIndexTabletState::LoadState(
config.GetReadAheadCacheMaxHandlesPerNode());
Impl->NodeIndexCache.Reset(config.GetNodeIndexCacheMaxNodes());
Impl->InMemoryIndexState.Reset(
config.GetInMemoryIndexCacheNodesCapacity(),
config.GetInMemoryIndexCacheNodeAttrsCapacity(),
config.GetInMemoryIndexCacheNodeRefsCapacity());
CalculateInMemoryIndexCacheCapacity(
config.GetInMemoryIndexCacheNodesCapacity(),
GetNodesCount(),
config.GetInMemoryIndexCacheNodesToNodesCapacityRatio()),
CalculateInMemoryIndexCacheCapacity(
config.GetInMemoryIndexCacheNodeAttrsCapacity(),
GetNodesCount(),
config.GetInMemoryIndexCacheNodesToNodeAttrsCapacityRatio()),
CalculateInMemoryIndexCacheCapacity(
config.GetInMemoryIndexCacheNodeRefsCapacity(),
GetNodesCount(),
config.GetInMemoryIndexCacheNodesToNodeRefsCapacityRatio()));

for (const auto& deletionMarker: largeDeletionMarkers) {
Impl->LargeBlocks.AddDeletionMarker(deletionMarker);
Expand Down
3 changes: 3 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_state_cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,11 @@ TInMemoryIndexStateStats TInMemoryIndexState::GetStats() const
{
return TInMemoryIndexStateStats{
.NodesCount = Nodes.size(),
.NodesCapacity = NodesCapacity,
.NodeRefsCount = NodeRefs.size(),
.NodeRefsCapacity = NodeRefsCapacity,
.NodeAttrsCount = NodeAttrs.size(),
.NodeAttrsCapacity = NodeAttrsCapacity,
.IsNodeRefsExhaustive = IsNodeRefsExhaustive,
};
}
Expand Down
3 changes: 3 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_state_cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@ namespace NCloud::NFileStore::NStorage {
struct TInMemoryIndexStateStats
{
ui64 NodesCount;
ui64 NodesCapacity;
ui64 NodeRefsCount;
ui64 NodeRefsCapacity;
ui64 NodeAttrsCount;
ui64 NodeAttrsCapacity;
bool IsNodeRefsExhaustive;
};

Expand Down
46 changes: 46 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_ut_cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1016,6 +1016,52 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_NodesCache)
statsAfter.ROCacheMissCount - statsBefore.ROCacheMissCount);
UNIT_ASSERT(!statsAfter.IsExhaustive);
}

Y_UNIT_TEST(ShouldCalculateCacheCapacityBasedOnRatio)
{
NProto::TStorageConfig storageConfig;
storageConfig.SetInMemoryIndexCacheEnabled(true);
// max(1000, 1024 / 10) = 1000
storageConfig.SetInMemoryIndexCacheNodesCapacity(1000);
storageConfig.SetInMemoryIndexCacheNodesToNodesCapacityRatio(10);

storageConfig.SetInMemoryIndexCacheNodeRefsCapacity(20);
storageConfig.SetInMemoryIndexCacheNodesToNodeRefsCapacityRatio(0);

// max(100, 1024 / 8) = 128
storageConfig.SetInMemoryIndexCacheNodeAttrsCapacity(100);
storageConfig.SetInMemoryIndexCacheNodesToNodeAttrsCapacityRatio(8);

TTestEnv env({}, storageConfig);
env.CreateSubDomain("nfs");

ui32 nodeIdx = env.CreateNode("nfs");
ui64 tabletId = env.BootIndexTablet(nodeIdx);

TIndexTabletClient tablet(
env.GetRuntime(),
nodeIdx,
tabletId,
TFileSystemConfig{.NodeCount = 1024});
tablet.RebootTablet();

tablet.SendRequest(tablet.CreateUpdateCounters());
env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1));

TTestRegistryVisitor visitor;
env.GetRegistry()->Visit(TInstant::Zero(), visitor);
visitor.ValidateExpectedCounters({
{{{"filesystem", "test"},
{"sensor", "InMemoryIndexStateNodesCapacity"}},
1000},
{{{"filesystem", "test"},
{"sensor", "InMemoryIndexStateNodeRefsCapacity"}},
20},
{{{"filesystem", "test"},
{"sensor", "InMemoryIndexStateNodeAttrsCapacity"}},
128},
});
}
}

} // namespace NCloud::NFileStore::NStorage

0 comments on commit d47444e

Please sign in to comment.