From a4b4bddd3a8ba743c7665932071f380f5e93fe55 Mon Sep 17 00:00:00 2001 From: Ganesh Vanahalli Date: Wed, 20 Dec 2023 20:28:47 +0530 Subject: [PATCH 1/3] Add a metric for memory used by the replay binary WASM in JIT --- arbitrator/jit/src/machine.rs | 3 ++- arbitrator/jit/src/main.rs | 3 ++- validator/server_jit/jit_machine.go | 33 +++++++++++++++++++------- validator/server_jit/machine_loader.go | 12 ++++++---- validator/server_jit/spawner.go | 10 ++++++-- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/arbitrator/jit/src/machine.rs b/arbitrator/jit/src/machine.rs index ed22e12ef9..c9119dd16e 100644 --- a/arbitrator/jit/src/machine.rs +++ b/arbitrator/jit/src/machine.rs @@ -280,7 +280,7 @@ impl WasmEnv { Ok(env) } - pub fn send_results(&mut self, error: Option) { + pub fn send_results(&mut self, error: Option, memory_used: u64) { let writer = match &mut self.process.socket { Some((writer, _)) => writer, None => return, @@ -307,6 +307,7 @@ impl WasmEnv { check!(socket::write_u64(writer, self.small_globals[1])); check!(socket::write_bytes32(writer, &self.large_globals[0])); check!(socket::write_bytes32(writer, &self.large_globals[1])); + check!(socket::write_u64(writer, memory_used)); check!(writer.flush()); } } diff --git a/arbitrator/jit/src/main.rs b/arbitrator/jit/src/main.rs index 513cd067c4..968da2a978 100644 --- a/arbitrator/jit/src/main.rs +++ b/arbitrator/jit/src/main.rs @@ -114,8 +114,9 @@ fn main() { true => None, false => Some(message), }; + let memory_used = memory.size().0 as u64 * 65_536; - env.send_results(error); + env.send_results(error, memory_used); } // require a usize be at least 32 bits wide diff --git a/validator/server_jit/jit_machine.go b/validator/server_jit/jit_machine.go index f763ce3ea0..a41e249cdb 100644 --- a/validator/server_jit/jit_machine.go +++ b/validator/server_jit/jit_machine.go @@ -16,17 +16,21 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/offchainlabs/nitro/util/arbmath" "github.com/offchainlabs/nitro/validator" ) +var jitWasmMemoryUsage = metrics.NewRegisteredHistogram("jit/wasm/memoryusage", nil, metrics.NewBoundedHistogramSample()) + type JitMachine struct { - binary string - process *exec.Cmd - stdin io.WriteCloser + binary string + process *exec.Cmd + stdin io.WriteCloser + wasmMemoryUsageLimit int } -func createJitMachine(jitBinary string, binaryPath string, cranelift bool, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { +func createJitMachine(jitBinary string, binaryPath string, cranelift bool, wasmMemoryUsageLimit int, moduleRoot common.Hash, fatalErrChan chan error) (*JitMachine, error) { invocation := []string{"--binary", binaryPath, "--forks"} if cranelift { invocation = append(invocation, "--cranelift") @@ -45,9 +49,10 @@ func createJitMachine(jitBinary string, binaryPath string, cranelift bool, modul }() machine := &JitMachine{ - binary: binaryPath, - process: process, - stdin: stdin, + binary: binaryPath, + process: process, + stdin: stdin, + wasmMemoryUsageLimit: wasmMemoryUsageLimit, } return machine, nil } @@ -258,8 +263,18 @@ func (machine *JitMachine) prove( if state.BlockHash, err = readHash(); err != nil { return state, err } - state.SendRoot, err = readHash() - return state, err + if state.SendRoot, err = readHash(); err != nil { + return state, err + } + memoryUsed, err := readUint64() + if err != nil { + return state, fmt.Errorf("failed to read memory usage from Jit machine: %w", err) + } + if memoryUsed > uint64(machine.wasmMemoryUsageLimit) { + log.Warn("memory used by jit wasm exceeds the wasm memory usage limit", "limit", machine.wasmMemoryUsageLimit, "memoryUsed", memoryUsed) + } + jitWasmMemoryUsage.Update(int64(memoryUsed)) + return state, nil default: message := "inter-process communication failure" log.Error("Jit Machine Failure", "message", message) diff --git a/validator/server_jit/machine_loader.go b/validator/server_jit/machine_loader.go index 5705a9a387..3a831928b7 100644 --- a/validator/server_jit/machine_loader.go +++ b/validator/server_jit/machine_loader.go @@ -13,13 +13,15 @@ import ( ) type JitMachineConfig struct { - ProverBinPath string - JitCranelift bool + ProverBinPath string + JitCranelift bool + WasmMemoryUsageLimit int } var DefaultJitMachineConfig = JitMachineConfig{ - JitCranelift: true, - ProverBinPath: "replay.wasm", + JitCranelift: true, + ProverBinPath: "replay.wasm", + WasmMemoryUsageLimit: 4294967296, } func getJitPath() (string, error) { @@ -57,7 +59,7 @@ func NewJitMachineLoader(config *JitMachineConfig, locator *server_common.Machin } createMachineThreadFunc := func(ctx context.Context, moduleRoot common.Hash) (*JitMachine, error) { binPath := filepath.Join(locator.GetMachinePath(moduleRoot), config.ProverBinPath) - return createJitMachine(jitPath, binPath, config.JitCranelift, moduleRoot, fatalErrChan) + return createJitMachine(jitPath, binPath, config.JitCranelift, config.WasmMemoryUsageLimit, moduleRoot, fatalErrChan) } return &JitMachineLoader{ MachineLoader: *server_common.NewMachineLoader[JitMachine](locator, createMachineThreadFunc), diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index ff1749506a..6489821b5b 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -18,18 +18,23 @@ import ( type JitSpawnerConfig struct { Workers int `koanf:"workers" reload:"hot"` Cranelift bool `koanf:"cranelift"` + + // TODO: change WasmMemoryUsageLimit to a string and use resourcemanager.ParseMemLimit + WasmMemoryUsageLimit int `koanf:"wasm-memory-usage-limit"` } type JitSpawnerConfigFecher func() *JitSpawnerConfig var DefaultJitSpawnerConfig = JitSpawnerConfig{ - Workers: 0, - Cranelift: true, + Workers: 0, + Cranelift: true, + WasmMemoryUsageLimit: 4294967296, // 2^32 WASM memeory limit } func JitSpawnerConfigAddOptions(prefix string, f *flag.FlagSet) { f.Int(prefix+".workers", DefaultJitSpawnerConfig.Workers, "number of concurrent validation threads") f.Bool(prefix+".cranelift", DefaultJitSpawnerConfig.Cranelift, "use Cranelift instead of LLVM when validating blocks using the jit-accelerated block validator") + f.Int(prefix+".wasm-memory-usage-limit", DefaultJitSpawnerConfig.WasmMemoryUsageLimit, "if memory used by a jit wasm exceeds this limit, a warning is logged") } type JitSpawner struct { @@ -44,6 +49,7 @@ func NewJitSpawner(locator *server_common.MachineLocator, config JitSpawnerConfi // TODO - preload machines machineConfig := DefaultJitMachineConfig machineConfig.JitCranelift = config().Cranelift + machineConfig.WasmMemoryUsageLimit = config().WasmMemoryUsageLimit loader, err := NewJitMachineLoader(&machineConfig, locator, fatalErrChan) if err != nil { return nil, err From e8ec476514ba6b9286380eb2d8190296c628767a Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Thu, 4 Jan 2024 17:55:21 -0700 Subject: [PATCH 2/3] Make clippy happy --- arbitrator/prover/src/machine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 0849312f3d..6ca552d83c 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -362,7 +362,7 @@ impl Module { bin.memories.len() <= 1, "Multiple memories are not supported" ); - if let Some(limits) = bin.memories.get(0) { + if let Some(limits) = bin.memories.first() { let page_size = Memory::PAGE_SIZE; let initial = limits.initial; // validate() checks this is less than max::u32 let allowed = u32::MAX as u64 / Memory::PAGE_SIZE - 1; // we require the size remain *below* 2^32 From b9e9bcfecc6f857e20bb9b46d18847927af57a35 Mon Sep 17 00:00:00 2001 From: Joshua Colvin Date: Thu, 4 Jan 2024 18:22:06 -0700 Subject: [PATCH 3/3] Make clippy happy --- arbitrator/jit/src/syscall.rs | 6 +++--- arbitrator/wasm-libraries/go-stub/src/lib.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arbitrator/jit/src/syscall.rs b/arbitrator/jit/src/syscall.rs index c81641a7f8..4f657eeefa 100644 --- a/arbitrator/jit/src/syscall.rs +++ b/arbitrator/jit/src/syscall.rs @@ -337,7 +337,7 @@ pub fn js_value_call(mut env: WasmEnvMut, sp: u32) -> MaybeEscape { let value = match (object, method_name.as_slice()) { (Ref(GO_ID), b"_makeFuncWrapper") => { - let arg = match args.get(0) { + let arg = match args.first() { Some(arg) => arg, None => fail!( "Go trying to call Go._makeFuncWrapper with bad args {:?}", @@ -415,7 +415,7 @@ pub fn js_value_call(mut env: WasmEnvMut, sp: u32) -> MaybeEscape { (Ref(CRYPTO_ID), b"getRandomValues") => { let name = "crypto.getRandomValues"; - let id = match args.get(0) { + let id = match args.first() { Some(Ref(x)) => x, _ => fail!("Go trying to call {name} with bad args {:?}", args), }; @@ -456,7 +456,7 @@ pub fn js_value_new(mut env: WasmEnvMut, sp: u32) { let args_len = sp.read_u64(2); let args = sp.read_value_slice(args_ptr, args_len); match class { - UINT8_ARRAY_ID => match args.get(0) { + UINT8_ARRAY_ID => match args.first() { Some(JsValue::Number(size)) => { let id = pool.insert(DynamicObject::Uint8Array(vec![0; *size as usize])); sp.write_u64(4, GoValue::Object(id).encode()); diff --git a/arbitrator/wasm-libraries/go-stub/src/lib.rs b/arbitrator/wasm-libraries/go-stub/src/lib.rs index df77893fcb..1a5d1963c7 100644 --- a/arbitrator/wasm-libraries/go-stub/src/lib.rs +++ b/arbitrator/wasm-libraries/go-stub/src/lib.rs @@ -218,7 +218,7 @@ pub unsafe extern "C" fn go__syscall_js_valueNew(sp: GoStack) { let args_len = sp.read_u64(2); let args = read_value_slice(args_ptr, args_len); if class == UINT8_ARRAY_ID { - if let Some(InterpValue::Number(size)) = args.get(0) { + if let Some(InterpValue::Number(size)) = args.first() { let id = DynamicObjectPool::singleton() .insert(DynamicObject::Uint8Array(vec![0; *size as usize])); sp.write_u64(4, GoValue::Object(id).encode()); @@ -321,7 +321,7 @@ unsafe fn value_call_impl(sp: &mut GoStack) -> Result { let args_len = sp.read_u64(4); let args = read_value_slice(args_ptr, args_len); if object == InterpValue::Ref(GO_ID) && &method_name == b"_makeFuncWrapper" { - let id = args.get(0).ok_or_else(|| { + let id = args.first().ok_or_else(|| { format!( "Go attempting to call Go._makeFuncWrapper with bad args {:?}", args, @@ -405,7 +405,7 @@ unsafe fn value_call_impl(sp: &mut GoStack) -> Result { )) } } else if object == InterpValue::Ref(CRYPTO_ID) && &method_name == b"getRandomValues" { - let id = match args.get(0) { + let id = match args.first() { Some(InterpValue::Ref(x)) => *x, _ => { return Err(format!(