diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto index 14021d32eec..7de7a2f12e9 100644 --- a/cloud/filestore/config/storage.proto +++ b/cloud/filestore/config/storage.proto @@ -360,11 +360,15 @@ message TStorageConfig optional bool InMemoryIndexCacheEnabled = 371; // Capacity of in-memory index cache, in number of entries per each table optional uint64 InMemoryIndexCacheNodesCapacity = 372; - reserved 373; // InMemoryIndexCacheNodesVerCapacity + // Ratio of maximal number of nodes in the fs to the in-memory index cache + // capacity. I. e., if there are max 1000 nodes and ratio is set to 10, + // then there can be max 100 entries in cache. If both capacity and ratio + // are set, the greater value is used. + optional uint64 InMemoryIndexCacheNodesToNodesCapacityRatio = 373; optional uint64 InMemoryIndexCacheNodeAttrsCapacity = 374; - reserved 375; // InMemoryIndexCacheNodeAttrsVerCapacity + optional uint64 InMemoryIndexCacheNodesToNodeAttrsCapacityRatio = 375; optional uint64 InMemoryIndexCacheNodeRefsCapacity = 376; - reserved 377; // InMemoryIndexCacheNodeRefsVerCapacity + optional uint64 InMemoryIndexCacheNodesToNodeRefsCapacityRatio = 377; // Used to send non-network metrics as network ones to HIVE, // while we use them for load balancing diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp index f2ed2732429..f36a5667444 100644 --- a/cloud/filestore/libs/storage/core/config.cpp +++ b/cloud/filestore/libs/storage/core/config.cpp @@ -206,8 +206,11 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases; \ xxx(InMemoryIndexCacheEnabled, bool, false )\ xxx(InMemoryIndexCacheNodesCapacity, ui64, 0 )\ + xxx(InMemoryIndexCacheNodesToNodesCapacityRatio, ui64, 0 )\ xxx(InMemoryIndexCacheNodeAttrsCapacity, ui64, 0 )\ + xxx(InMemoryIndexCacheNodesToNodeAttrsCapacityRatio,ui64, 0 )\ xxx(InMemoryIndexCacheNodeRefsCapacity, ui64, 0 )\ + xxx(InMemoryIndexCacheNodesToNodeRefsCapacityRatio, ui64, 0 )\ xxx(InMemoryIndexCacheLoadOnTabletStart, bool, false )\ xxx(InMemoryIndexCacheLoadOnTabletStartRowsPerTx, ui64, 1000000 )\ \ diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h index c86f3ba25f6..58382718be2 100644 --- a/cloud/filestore/libs/storage/core/config.h +++ b/cloud/filestore/libs/storage/core/config.h @@ -233,8 +233,11 @@ class TStorageConfig bool GetInMemoryIndexCacheEnabled() const; ui64 GetInMemoryIndexCacheNodesCapacity() const; + ui64 GetInMemoryIndexCacheNodesToNodesCapacityRatio() const; ui64 GetInMemoryIndexCacheNodeAttrsCapacity() const; + ui64 GetInMemoryIndexCacheNodesToNodeAttrsCapacityRatio() const; ui64 GetInMemoryIndexCacheNodeRefsCapacity() const; + ui64 GetInMemoryIndexCacheNodesToNodeRefsCapacityRatio() const; bool GetInMemoryIndexCacheLoadOnTabletStart() const; ui64 GetInMemoryIndexCacheLoadOnTabletStartRowsPerTx() const; diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index fe3f12e2f5b..9749648df9b 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -115,8 +115,11 @@ class TIndexTabletActor final std::atomic InMemoryIndexStateRWCount{0}; std::atomic InMemoryIndexStateNodesCount; + std::atomic InMemoryIndexStateNodesCapacity; std::atomic InMemoryIndexStateNodeRefsCount; + std::atomic InMemoryIndexStateNodeRefsCapacity; std::atomic InMemoryIndexStateNodeAttrsCount; + std::atomic InMemoryIndexStateNodeAttrsCapacity; std::atomic InMemoryIndexStateIsExhaustive; // Data stats diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp index 8d8a0f49aee..7491292b97b 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_counters.cpp @@ -291,12 +291,21 @@ void TIndexTabletActor::TMetrics::Register( REGISTER_AGGREGATABLE_SUM( InMemoryIndexStateNodesCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM( + InMemoryIndexStateNodesCapacity, + EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM( InMemoryIndexStateNodeRefsCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM( + InMemoryIndexStateNodeRefsCapacity, + EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM( InMemoryIndexStateNodeAttrsCount, EMetricType::MT_ABSOLUTE); + REGISTER_AGGREGATABLE_SUM( + InMemoryIndexStateNodeAttrsCapacity, + EMetricType::MT_ABSOLUTE); REGISTER_AGGREGATABLE_SUM( InMemoryIndexStateIsExhaustive, EMetricType::MT_ABSOLUTE); @@ -487,8 +496,11 @@ void TIndexTabletActor::TMetrics::Update( Store(NodeIndexCacheNodeCount, nodeIndexCacheStats.NodeCount); Store(InMemoryIndexStateNodesCount, inMemoryIndexStateStats.NodesCount); + Store(InMemoryIndexStateNodesCapacity, inMemoryIndexStateStats.NodesCapacity); Store(InMemoryIndexStateNodeRefsCount, inMemoryIndexStateStats.NodeRefsCount); + Store(InMemoryIndexStateNodeRefsCapacity, inMemoryIndexStateStats.NodeRefsCapacity); Store(InMemoryIndexStateNodeAttrsCount, inMemoryIndexStateStats.NodeAttrsCount); + Store(InMemoryIndexStateNodeAttrsCapacity, inMemoryIndexStateStats.NodeAttrsCapacity); Store(InMemoryIndexStateIsExhaustive, inMemoryIndexStateStats.IsNodeRefsExhaustive); Store( diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.cpp b/cloud/filestore/libs/storage/tablet/tablet_state.cpp index 9531ba9d22d..78b5aed61c1 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state.cpp @@ -59,6 +59,17 @@ bool IsValid(const NProto::TFileStorePerformanceProfile& profile) && profile.GetDefaultPostponedRequestWeight(); } +ui64 CalculateInMemoryIndexCacheCapacity( + const ui64 capacity, + const ui64 maxNodes, + const ui64 nodesToCapacityRatio) +{ + if (nodesToCapacityRatio == 0) { + return capacity; + } + return Max(capacity, maxNodes / nodesToCapacityRatio); +} + } // namespace //////////////////////////////////////////////////////////////////////////////// @@ -126,9 +137,18 @@ void TIndexTabletState::LoadState( config.GetReadAheadCacheMaxHandlesPerNode()); Impl->NodeIndexCache.Reset(config.GetNodeIndexCacheMaxNodes()); Impl->InMemoryIndexState.Reset( - config.GetInMemoryIndexCacheNodesCapacity(), - config.GetInMemoryIndexCacheNodeAttrsCapacity(), - config.GetInMemoryIndexCacheNodeRefsCapacity()); + CalculateInMemoryIndexCacheCapacity( + config.GetInMemoryIndexCacheNodesCapacity(), + GetNodesCount(), + config.GetInMemoryIndexCacheNodesToNodesCapacityRatio()), + CalculateInMemoryIndexCacheCapacity( + config.GetInMemoryIndexCacheNodeAttrsCapacity(), + GetNodesCount(), + config.GetInMemoryIndexCacheNodesToNodeAttrsCapacityRatio()), + CalculateInMemoryIndexCacheCapacity( + config.GetInMemoryIndexCacheNodeRefsCapacity(), + GetNodesCount(), + config.GetInMemoryIndexCacheNodesToNodeRefsCapacityRatio())); for (const auto& deletionMarker: largeDeletionMarkers) { Impl->LargeBlocks.AddDeletionMarker(deletionMarker); diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_cache.cpp b/cloud/filestore/libs/storage/tablet/tablet_state_cache.cpp index 1267b35f4ac..37118e6ba18 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_cache.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state_cache.cpp @@ -45,8 +45,11 @@ TInMemoryIndexStateStats TInMemoryIndexState::GetStats() const { return TInMemoryIndexStateStats{ .NodesCount = Nodes.size(), + .NodesCapacity = NodesCapacity, .NodeRefsCount = NodeRefs.size(), + .NodeRefsCapacity = NodeRefsCapacity, .NodeAttrsCount = NodeAttrs.size(), + .NodeAttrsCapacity = NodeAttrsCapacity, .IsNodeRefsExhaustive = IsNodeRefsExhaustive, }; } diff --git a/cloud/filestore/libs/storage/tablet/tablet_state_cache.h b/cloud/filestore/libs/storage/tablet/tablet_state_cache.h index 4bdd251fbb5..0de06e25c7e 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state_cache.h +++ b/cloud/filestore/libs/storage/tablet/tablet_state_cache.h @@ -13,8 +13,11 @@ namespace NCloud::NFileStore::NStorage { struct TInMemoryIndexStateStats { ui64 NodesCount; + ui64 NodesCapacity; ui64 NodeRefsCount; + ui64 NodeRefsCapacity; ui64 NodeAttrsCount; + ui64 NodeAttrsCapacity; bool IsNodeRefsExhaustive; }; diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_cache.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_cache.cpp index 5c69eefb7cc..69655988f56 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_cache.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_cache.cpp @@ -1016,6 +1016,52 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_NodesCache) statsAfter.ROCacheMissCount - statsBefore.ROCacheMissCount); UNIT_ASSERT(!statsAfter.IsExhaustive); } + + Y_UNIT_TEST(ShouldCalculateCacheCapacityBasedOnRatio) + { + NProto::TStorageConfig storageConfig; + storageConfig.SetInMemoryIndexCacheEnabled(true); + // max(1000, 1024 / 10) = 1000 + storageConfig.SetInMemoryIndexCacheNodesCapacity(1000); + storageConfig.SetInMemoryIndexCacheNodesToNodesCapacityRatio(10); + + storageConfig.SetInMemoryIndexCacheNodeRefsCapacity(20); + storageConfig.SetInMemoryIndexCacheNodesToNodeRefsCapacityRatio(0); + + // max(100, 1024 / 8) = 128 + storageConfig.SetInMemoryIndexCacheNodeAttrsCapacity(100); + storageConfig.SetInMemoryIndexCacheNodesToNodeAttrsCapacityRatio(8); + + TTestEnv env({}, storageConfig); + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + ui64 tabletId = env.BootIndexTablet(nodeIdx); + + TIndexTabletClient tablet( + env.GetRuntime(), + nodeIdx, + tabletId, + TFileSystemConfig{.NodeCount = 1024}); + tablet.RebootTablet(); + + tablet.SendRequest(tablet.CreateUpdateCounters()); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1)); + + TTestRegistryVisitor visitor; + env.GetRegistry()->Visit(TInstant::Zero(), visitor); + visitor.ValidateExpectedCounters({ + {{{"filesystem", "test"}, + {"sensor", "InMemoryIndexStateNodesCapacity"}}, + 1000}, + {{{"filesystem", "test"}, + {"sensor", "InMemoryIndexStateNodeRefsCapacity"}}, + 20}, + {{{"filesystem", "test"}, + {"sensor", "InMemoryIndexStateNodeAttrsCapacity"}}, + 128}, + }); + } } } // namespace NCloud::NFileStore::NStorage