From 88bce924d7dd1bd219632262c0d531c7946a0067 Mon Sep 17 00:00:00 2001 From: Liang Gong Date: Tue, 20 Aug 2024 17:40:07 -0700 Subject: [PATCH] fix(core): Use lower threshold for NumericDictionary fast store (#125) (#125) Summary: User lower fast store size threhosld for NumericDictionary as per OSS feedback: https://github.com/facebook/memlab/issues/125 Closes #125 Reviewed By: twobassdrum Differential Revision: D61559911 fbshipit-source-id: 91eda9d9d04f9cdc8e9b339c26a8612c34cd82e2 --- packages/cli/src/commands/MemLabRunCommand.ts | 2 + .../cli/src/commands/heap/CheckLeakCommand.ts | 2 + .../cli/src/commands/heap/DiffLeakCommand.ts | 2 + .../commands/heap/GetRetainerTraceCommand.ts | 2 + .../src/commands/heap/HeapAnalysisCommand.ts | 6 ++- .../interactive/InteractiveHeapCommand.ts | 7 ++- .../InteractiveHeapExploreCommand.ts | 2 + .../heap/HeapParserDictFastStoreSizeOption.ts | 48 +++++++++++++++++++ .../cli/src/options/lib/OptionConstant.ts | 1 + packages/core/src/lib/Config.ts | 4 ++ .../core/src/lib/heap-data/HeapSnapshot.ts | 4 +- .../lib/heap-data/utils/NumericDictionary.ts | 27 +++++++++-- website/docs/cli/CLI-commands.md | 6 +++ 13 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 packages/cli/src/options/heap/HeapParserDictFastStoreSizeOption.ts diff --git a/packages/cli/src/commands/MemLabRunCommand.ts b/packages/cli/src/commands/MemLabRunCommand.ts index 46981aae2..7c9af686e 100644 --- a/packages/cli/src/commands/MemLabRunCommand.ts +++ b/packages/cli/src/commands/MemLabRunCommand.ts @@ -32,6 +32,7 @@ import TargetFileOption from '../options/heap/TargetFileOption'; import FinalFileOption from '../options/heap/FinalFileOption'; import SnapshotDirectoryOption from '../options/heap/SnapshotDirectoryOption'; import JSEngineOption from '../options/heap/JSEngineOption'; +import HeapParserDictFastStoreSizeOption from '../options/heap/HeapParserDictFastStoreSizeOption'; export default class MemLabRunCommand extends BaseCommand { getCommandName(): string { @@ -73,6 +74,7 @@ export default class MemLabRunCommand extends BaseCommand { new FinalFileOption(), new SnapshotDirectoryOption(), new JSEngineOption(), + new HeapParserDictFastStoreSizeOption(), ]; } diff --git a/packages/cli/src/commands/heap/CheckLeakCommand.ts b/packages/cli/src/commands/heap/CheckLeakCommand.ts index 1c577d24c..6ea8fa90e 100644 --- a/packages/cli/src/commands/heap/CheckLeakCommand.ts +++ b/packages/cli/src/commands/heap/CheckLeakCommand.ts @@ -35,6 +35,7 @@ import MLClusteringMaxDFOption from '../../options/MLClusteringMaxDFOption'; import CleanupSnapshotOption from '../../options/heap/CleanupSnapshotOption'; import SetWorkingDirectoryOption from '../../options/SetWorkingDirectoryOption'; import OptionConstant from '../../options/lib/OptionConstant'; +import HeapParserDictFastStoreSizeOption from '../../options/heap/HeapParserDictFastStoreSizeOption'; export type CheckLeakCommandOptions = { isMLClustering?: boolean; @@ -148,6 +149,7 @@ or option 2 mentioned above). new MLClusteringMaxDFOption(), new CleanupSnapshotOption(), new SetWorkingDirectoryOption(), + new HeapParserDictFastStoreSizeOption(), ]; } diff --git a/packages/cli/src/commands/heap/DiffLeakCommand.ts b/packages/cli/src/commands/heap/DiffLeakCommand.ts index c529eff70..2de772b11 100644 --- a/packages/cli/src/commands/heap/DiffLeakCommand.ts +++ b/packages/cli/src/commands/heap/DiffLeakCommand.ts @@ -30,6 +30,7 @@ import SetMaxClusterSampleSizeOption from '../../options/SetMaxClusterSampleSize import SetTraceContainsFilterOption from '../../options/heap/SetTraceContainsFilterOption'; import SetControlSnapshotOption from '../../options/experiment/SetControlSnapshotOption'; import SetTreatmentSnapshotOption from '../../options/experiment/SetTreatmentSnapshotOption'; +import HeapParserDictFastStoreSizeOption from '../../options/heap/HeapParserDictFastStoreSizeOption'; export type WorkDirSettings = { controlWorkDirs: Array; @@ -91,6 +92,7 @@ export default class CheckLeakCommand extends BaseCommand { new MLClusteringMaxDFOption(), new SetMaxClusterSampleSizeOption(), new SetTraceContainsFilterOption(), + new HeapParserDictFastStoreSizeOption(), ]; } diff --git a/packages/cli/src/commands/heap/GetRetainerTraceCommand.ts b/packages/cli/src/commands/heap/GetRetainerTraceCommand.ts index e566e3df5..43d0c1501 100644 --- a/packages/cli/src/commands/heap/GetRetainerTraceCommand.ts +++ b/packages/cli/src/commands/heap/GetRetainerTraceCommand.ts @@ -18,6 +18,7 @@ import JSEngineOption from '../../options/heap/JSEngineOption'; import HeapNodeIdOption from '../../options/heap/HeapNodeIdOption'; import SnapshotDirectoryOption from '../../options/heap/SnapshotDirectoryOption'; import {fileManager} from '@memlab/core'; +import HeapParserDictFastStoreSizeOption from '../../options/heap/HeapParserDictFastStoreSizeOption'; async function calculateRetainerTrace(): Promise { const snapshotPath = utils.getSingleSnapshotFileForAnalysis(); @@ -51,6 +52,7 @@ export default class GetRetainerTraceCommand extends BaseCommand { new SnapshotDirectoryOption(), new JSEngineOption(), new HeapNodeIdOption().required(), + new HeapParserDictFastStoreSizeOption(), ]; } diff --git a/packages/cli/src/commands/heap/HeapAnalysisCommand.ts b/packages/cli/src/commands/heap/HeapAnalysisCommand.ts index 4938d302e..dc0e48389 100644 --- a/packages/cli/src/commands/heap/HeapAnalysisCommand.ts +++ b/packages/cli/src/commands/heap/HeapAnalysisCommand.ts @@ -19,6 +19,7 @@ import HelperCommand from '../helper/HelperCommand'; import InitDirectoryCommand from '../InitDirectoryCommand'; import HeapAnalysisPluginOption from '../../options/heap/HeapAnalysisPluginOption'; import {ParsedArgs} from 'minimist'; +import HeapParserDictFastStoreSizeOption from '../../options/heap/HeapParserDictFastStoreSizeOption'; export default class RunHeapAnalysisCommand extends BaseCommand { getCommandName(): string { @@ -38,7 +39,10 @@ export default class RunHeapAnalysisCommand extends BaseCommand { } getOptions(): BaseOption[] { - return [new HeapAnalysisPluginOption()]; + return [ + new HeapAnalysisPluginOption(), + new HeapParserDictFastStoreSizeOption(), + ]; } getSubCommands(): BaseCommand[] { diff --git a/packages/cli/src/commands/heap/interactive/InteractiveHeapCommand.ts b/packages/cli/src/commands/heap/interactive/InteractiveHeapCommand.ts index 0eca16c15..1d20d1b24 100644 --- a/packages/cli/src/commands/heap/interactive/InteractiveHeapCommand.ts +++ b/packages/cli/src/commands/heap/interactive/InteractiveHeapCommand.ts @@ -27,6 +27,7 @@ import {fileManager} from '@memlab/core'; import {heapConfig, loadHeapSnapshot} from '@memlab/heap-analysis'; import {CommandDispatcher} from '../../../Dispatcher'; import InteractiveCommandLoader from './InteractiveCommandLoader'; +import HeapParserDictFastStoreSizeOption from '../../../options/heap/HeapParserDictFastStoreSizeOption'; export default class InteractiveHeapCommand extends BaseCommand { getCommandName(): string { @@ -46,7 +47,11 @@ export default class InteractiveHeapCommand extends BaseCommand { } getOptions(): BaseOption[] { - return [new SnapshotFileOption(), new JSEngineOption()]; + return [ + new SnapshotFileOption(), + new JSEngineOption(), + new HeapParserDictFastStoreSizeOption(), + ]; } private exitAttempt = 0; diff --git a/packages/cli/src/commands/heap/interactive/InteractiveHeapExploreCommand.ts b/packages/cli/src/commands/heap/interactive/InteractiveHeapExploreCommand.ts index 1534792cc..ddcfbd9a0 100644 --- a/packages/cli/src/commands/heap/interactive/InteractiveHeapExploreCommand.ts +++ b/packages/cli/src/commands/heap/interactive/InteractiveHeapExploreCommand.ts @@ -28,6 +28,7 @@ import CliScreen from './ui-components/CliScreen'; import HeapNodeIdOption from '../../../options/heap/HeapNodeIdOption'; import MLClusteringOption from '../../../options/MLClusteringOption'; import SetWorkingDirectoryOption from '../../../options/SetWorkingDirectoryOption'; +import HeapParserDictFastStoreSizeOption from '../../../options/heap/HeapParserDictFastStoreSizeOption'; export default class InteractiveHeapViewCommand extends BaseCommand { getCommandName(): string { @@ -53,6 +54,7 @@ export default class InteractiveHeapViewCommand extends BaseCommand { new HeapNodeIdOption(), new MLClusteringOption(), new SetWorkingDirectoryOption(), + new HeapParserDictFastStoreSizeOption(), ]; } diff --git a/packages/cli/src/options/heap/HeapParserDictFastStoreSizeOption.ts b/packages/cli/src/options/heap/HeapParserDictFastStoreSizeOption.ts new file mode 100644 index 000000000..b01c2f23d --- /dev/null +++ b/packages/cli/src/options/heap/HeapParserDictFastStoreSizeOption.ts @@ -0,0 +1,48 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + * @oncall web_perf_infra + */ + +import type {ParsedArgs} from 'minimist'; +import type {MemLabConfig} from '@memlab/core'; +import {BaseOption, utils} from '@memlab/core'; +import optionConstants from '../lib/OptionConstant'; + +export default class HeapParserDictFastStoreSizeOption extends BaseOption { + getOptionName(): string { + return optionConstants.optionNames.HEAP_PARSER_DICT_FAST_STORE_SIZE; + } + + getDescription(): string { + return ( + 'the size threshold for swtiching from fast store to slower store in ' + + 'the heap snapshot parser. The default value is 5,000,000. If you get ' + + 'the `FATAL ERROR: invalid table size Allocation failed - JavaScript ' + + 'heap out of memory` error, try to decrease the threshold here' + ); + } + + getExampleValues(): string[] { + return ['500000', '1000000']; + } + + async parse(config: MemLabConfig, args: ParsedArgs): Promise { + if (this.getOptionName() in args) { + const sizeThreshold = parseInt(args[this.getOptionName()], 10); + if (!isNaN(sizeThreshold)) { + if (sizeThreshold <= 0 || sizeThreshold > 10_000_000) { + utils.haltOrThrow( + `Invalid value for ${this.getOptionName()}: ${sizeThreshold}. ` + + 'Valid range is [1, 10_000_000]', + ); + } + config.heapParserDictFastStoreSize = sizeThreshold; + } + } + } +} diff --git a/packages/cli/src/options/lib/OptionConstant.ts b/packages/cli/src/options/lib/OptionConstant.ts index c63ff0603..68bd033e7 100644 --- a/packages/cli/src/options/lib/OptionConstant.ts +++ b/packages/cli/src/options/lib/OptionConstant.ts @@ -27,6 +27,7 @@ const optionNames = { FULL: 'full', HEADFUL: 'headful', HEAP_ANALYSIS_PLUGIN_FILE: 'analysis-plugin', + HEAP_PARSER_DICT_FAST_STORE_SIZE: 'heap-parser-dict-fast-store-size', HELP: 'help', IGNORE_LEAK_CLUSTER_SIZE_BELOW: 'ignore-leak-cluster-size-below', INTERACTION: 'interaction', diff --git a/packages/core/src/lib/Config.ts b/packages/core/src/lib/Config.ts index 97a58a812..f66410322 100644 --- a/packages/core/src/lib/Config.ts +++ b/packages/core/src/lib/Config.ts @@ -259,6 +259,7 @@ export class MemLabConfig { filterTraceByName: Nullable; skipBrowserCloseWait: boolean; simplifyCodeSerialization: boolean; + heapParserDictFastStoreSize: number; constructor(options: ConfigOption = {}) { // init properties, they can be configured manually @@ -508,6 +509,9 @@ export class MemLabConfig { // object size below the threshold won't be reported this.unboundSizeThreshold = 10 * 1024; + // threshold for swtiching between fast store and slower store + // of NumericDictionary used by heap snapshot parser + this.heapParserDictFastStoreSize = 5_000_000; // if true reset the GK list in visit synthesizer this.resetGK = false; // default userAgent, if undefined use puppeteer's default value diff --git a/packages/core/src/lib/heap-data/HeapSnapshot.ts b/packages/core/src/lib/heap-data/HeapSnapshot.ts index 33da65d2f..e11010c41 100644 --- a/packages/core/src/lib/heap-data/HeapSnapshot.ts +++ b/packages/core/src/lib/heap-data/HeapSnapshot.ts @@ -284,7 +284,9 @@ export default class HeapSnapshot implements IHeapSnapshot { _buildNodeIdx(): void { info.overwrite('building node index...'); - this._nodeId2NodeIdx = new NumericDictionary(this._nodeCount); + this._nodeId2NodeIdx = new NumericDictionary(this._nodeCount, { + fastStoreSize: config.heapParserDictFastStoreSize, + }); // iterate over each node const nodeValues = this.snapshot.nodes; const nodeFieldsCount = this._nodeFieldsCount; diff --git a/packages/core/src/lib/heap-data/utils/NumericDictionary.ts b/packages/core/src/lib/heap-data/utils/NumericDictionary.ts index d118ae203..17e1495d8 100644 --- a/packages/core/src/lib/heap-data/utils/NumericDictionary.ts +++ b/packages/core/src/lib/heap-data/utils/NumericDictionary.ts @@ -14,23 +14,40 @@ import {Nullable} from '../../Types'; // if the number is larger than this, we will use the slow store -const MAX_FAST_STORE_SIZE = 50_000_000; +const DEFAULT_FAST_STORE_SIZE = 5_000_000; +// maxmimum fast store size allowed to set to NumericDictionary, +// if the store size is bigger than this, it will cause exceptions +// in JS engines on some platforms +const MAX_FAST_STORE_SIZE = 10_000_000; type DirectMap = Map; type IndirectMap = Map>; +export type NumericDictOptions = { + fastStoreSize?: number; +}; + export default class NumericDictionary { private useFastStore = true; private fastStore: Nullable = null; private slowStore: Nullable = null; private numberOfShards = 1; + private fastStoreSize = DEFAULT_FAST_STORE_SIZE; - constructor(size: number) { - this.useFastStore = size <= MAX_FAST_STORE_SIZE; + constructor(size: number, options: NumericDictOptions = {}) { + if (options.fastStoreSize != null) { + if ( + options.fastStoreSize > 0 && + options.fastStoreSize <= MAX_FAST_STORE_SIZE + ) { + this.fastStoreSize = options.fastStoreSize; + } + } + this.useFastStore = size <= this.fastStoreSize; if (this.useFastStore) { this.fastStore = new Map(); } else { - this.numberOfShards = Math.ceil(size / MAX_FAST_STORE_SIZE); + this.numberOfShards = Math.ceil(size / this.fastStoreSize); this.slowStore = new Map(); } } @@ -40,7 +57,7 @@ export default class NumericDictionary { } getShard(key: number): number { - return Math.floor(key / MAX_FAST_STORE_SIZE); + return Math.floor(key / this.fastStoreSize); } has(key: number): boolean { diff --git a/website/docs/cli/CLI-commands.md b/website/docs/cli/CLI-commands.md index 99801b531..4a6b41968 100644 --- a/website/docs/cli/CLI-commands.md +++ b/website/docs/cli/CLI-commands.md @@ -105,6 +105,7 @@ memlab find-leaks --work-dir /memlab/working/dir/generated/by/memlab/ * **`--ml-clustering-max-df`**: set percentage based max document frequency for limiting the terms that appear too often * **`--clean-up-snapshot`**: clean up heap snapshots after running * **`--work-dir`**: set the working directory of the current run + * **`--heap-parser-dict-fast-store-size`**: the size threshold for swtiching from fast store to slower store in the heap snapshot parser. The default value is 5,000,000. If you get the `FATAL ERROR: invalid table size Allocation failed - JavaScript heap out of memory` error, try to decrease the threshold here * **`--help`**, **`-h`**: print helper text * **`--verbose`**, **`-v`**: show more details * **`--sc`**: set to continuous test mode @@ -135,6 +136,7 @@ memlab diff-leaks * **`--ml-clustering-max-df`**: set percentage based max document frequency for limiting the terms that appear too often * **`--max-cluster-sample-size`**: specify the max number of leak traces as input to leak trace clustering algorithm. Big sample size will preserve more complete inforrmation, but may risk out-of-memory crash. * **`--trace-contains`**: set the node name or edge name to filter leak traces that contain the name + * **`--heap-parser-dict-fast-store-size`**: the size threshold for swtiching from fast store to slower store in the heap snapshot parser. The default value is 5,000,000. If you get the `FATAL ERROR: invalid table size Allocation failed - JavaScript heap out of memory` error, try to decrease the threshold here * **`--work-dir`**: set the working directory of the current run * **`--help`**, **`-h`**: print helper text * **`--verbose`**, **`-v`**: show more details @@ -162,6 +164,7 @@ memlab trace --node-id=128127 * **`--snapshot-dir`**: set directory path containing all heap snapshots under analysis * **`--engine`**: set the JavaScript engine (default to V8) * **`--node-id`**: set heap node ID + * **`--heap-parser-dict-fast-store-size`**: the size threshold for swtiching from fast store to slower store in the heap snapshot parser. The default value is 5,000,000. If you get the `FATAL ERROR: invalid table size Allocation failed - JavaScript heap out of memory` error, try to decrease the threshold here * **`--help`**, **`-h`**: print helper text * **`--verbose`**, **`-v`**: show more details * **`--sc`**: set to continuous test mode @@ -178,6 +181,7 @@ memlab analyze [PLUGIN_OPTIONS] **Options**: * **`--analysis-plugin`**: specify the external heap analysis plugin file (must be a vanilla JS file ended with `Analysis.js` suffix) + * **`--heap-parser-dict-fast-store-size`**: the size threshold for swtiching from fast store to slower store in the heap snapshot parser. The default value is 5,000,000. If you get the `FATAL ERROR: invalid table size Allocation failed - JavaScript heap out of memory` error, try to decrease the threshold here * **`--work-dir`**: set the working directory of the current run * **`--help`**, **`-h`**: print helper text * **`--verbose`**, **`-v`**: show more details @@ -421,6 +425,7 @@ memlab heap --snapshot **Options**: * **`--snapshot`**: set file path of the heap snapshot under analysis * **`--engine`**: set the JavaScript engine (default to V8) + * **`--heap-parser-dict-fast-store-size`**: the size threshold for swtiching from fast store to slower store in the heap snapshot parser. The default value is 5,000,000. If you get the `FATAL ERROR: invalid table size Allocation failed - JavaScript heap out of memory` error, try to decrease the threshold here * **`--help`**, **`-h`**: print helper text * **`--verbose`**, **`-v`**: show more details * **`--sc`**: set to continuous test mode @@ -441,6 +446,7 @@ memlab view-heap --snapshot * **`--node-id`**: set heap node ID * **`--ml-clustering`**: use machine learning algorithms for clustering leak traces (by default, traces are clustered by heuristics) * **`--work-dir`**: set the working directory of the current run + * **`--heap-parser-dict-fast-store-size`**: the size threshold for swtiching from fast store to slower store in the heap snapshot parser. The default value is 5,000,000. If you get the `FATAL ERROR: invalid table size Allocation failed - JavaScript heap out of memory` error, try to decrease the threshold here * **`--help`**, **`-h`**: print helper text * **`--verbose`**, **`-v`**: show more details * **`--sc`**: set to continuous test mode