From e96a151f34ba36d66751277ae5b21457c3a47a99 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 8 Nov 2024 13:38:16 -0300 Subject: [PATCH 01/13] Fix discrepancy in interpreter --- src/interpreter.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/interpreter.rs b/src/interpreter.rs index 440508b3..981f923f 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -519,8 +519,6 @@ impl<'a, 'b, C: ContextObject> Interpreter<'a, 'b, C> { } check_pc!(self, next_pc, target_pc.wrapping_sub(self.program_vm_addr) / ebpf::INSN_SIZE as u64); if self.executable.get_sbpf_version().static_syscalls() && self.executable.get_function_registry().lookup_by_key(next_pc as u32).is_none() { - self.vm.due_insn_count += 1; - self.reg[11] = next_pc; throw_error!(self, EbpfError::UnsupportedInstruction); } }, From 58713347d72d76445d7c4ef5a48f43bb8c3001d5 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 8 Nov 2024 13:43:45 -0300 Subject: [PATCH 02/13] Introduce new Anchor --- src/jit.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/jit.rs b/src/jit.rs index a291a9e5..77e8b240 100644 --- a/src/jit.rs +++ b/src/jit.rs @@ -198,6 +198,7 @@ const ANCHOR_CALL_UNSUPPORTED_INSTRUCTION: usize = 10; const ANCHOR_EXTERNAL_FUNCTION_CALL: usize = 11; const ANCHOR_INTERNAL_FUNCTION_CALL_PROLOGUE: usize = 12; const ANCHOR_INTERNAL_FUNCTION_CALL_REG: usize = 13; +const ANCHOR_CALL_REG_UNSUPPORTED_INSTRUCTION: usize = 14; const ANCHOR_TRANSLATE_MEMORY_ADDRESS: usize = 21; const ANCHOR_COUNT: usize = 30; // Update me when adding or removing anchors @@ -1678,8 +1679,8 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { - mem::size_of::() as i32; // Jump from end of instruction unsafe { ptr::write_unaligned(jump.location as *mut i32, offset_value); } } - // There is no `VerifierError::JumpToMiddleOfLDDW` for `call imm` so patch it here - let call_unsupported_instruction = self.anchors[ANCHOR_CALL_UNSUPPORTED_INSTRUCTION] as usize; + // Patch addresses to which `callx` may raise an unsupported instruction error + let call_unsupported_instruction = self.anchors[ANCHOR_CALL_REG_UNSUPPORTED_INSTRUCTION] as usize; if self.executable.get_sbpf_version().static_syscalls() { let mut prev_pc = 0; for current_pc in self.executable.get_function_registry().keys() { From b0ea2f15c31bc39302cb7cc73374ca938631ff3d Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 8 Nov 2024 16:30:44 -0300 Subject: [PATCH 03/13] Handle call reg unsupported instruction correctly in jit --- src/jit.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/jit.rs b/src/jit.rs index 77e8b240..d4c1b25f 100644 --- a/src/jit.rs +++ b/src/jit.rs @@ -1550,6 +1550,10 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { // Outputs: Guest target pc in REGISTER_SCRATCH, Host target address in RIP self.set_anchor(ANCHOR_INTERNAL_FUNCTION_CALL_REG); self.emit_ins(X86Instruction::push(REGISTER_MAP[0], None)); + // REGISTER_SCRATCH contains the current program counter, and we must store it for proper + // error handling. We can discard the value if callx succeeds, so we are not incrementing + // the stack pointer (RSP). + self.emit_ins(X86Instruction::store(OperandSize::S64, REGISTER_SCRATCH, RSP, X86IndirectAccess::OffsetIndexShift(-8, RSP, 0))); self.emit_ins(X86Instruction::mov(OperandSize::S64, REGISTER_MAP[FRAME_PTR_REG], REGISTER_MAP[0])); // Calculate offset relative to program_vm_addr self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_MAP[FRAME_PTR_REG], self.program_vm_addr as i64)); @@ -1583,6 +1587,17 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_ins(X86Instruction::xchg(OperandSize::S64, REGISTER_MAP[0], RSP, Some(X86IndirectAccess::OffsetIndexShift(0, RSP, 0)))); // Swap REGISTER_MAP[0] and host_target_address self.emit_ins(X86Instruction::return_near()); // Tail call to host_target_address + // If callx lands in an invalid address, we must undo the changes in the instruction meter + // so that we can correctly calculate the number of executed instructions for error handling. + self.set_anchor(ANCHOR_CALL_REG_UNSUPPORTED_INSTRUCTION); + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x29, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter -= guest_target_pc + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, 1, None)); // instruction_meter += 1 + // Retrieve the current program from the stack. `return_near` popped an element from the stack, + // so the offset is 16. + self.emit_ins(X86Instruction::load(OperandSize::S64, RSP, REGISTER_SCRATCH, X86IndirectAccess::OffsetIndexShift(-16, RSP, 0))); + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x01, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter += guest_current_pc + self.emit_ins(X86Instruction::jump_immediate(self.relative_to_anchor(ANCHOR_CALL_UNSUPPORTED_INSTRUCTION, 5))); + // Translates a vm memory address to a host memory address for (access_type, len) in &[ (AccessType::Load, 1i32), From 47cc12aff41d61b32f0abae4872bb21618bad170 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 8 Nov 2024 16:32:00 -0300 Subject: [PATCH 04/13] Fix existing test --- tests/execution.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/execution.rs b/tests/execution.rs index 4c6b857c..a814654a 100644 --- a/tests/execution.rs +++ b/tests/execution.rs @@ -2347,6 +2347,7 @@ fn test_callx() { #[test] fn test_err_callx_unregistered() { + // We execute three instructions when callx errors out. test_interpreter_and_jit_asm!( " mov64 r0, 0x0 @@ -2356,7 +2357,7 @@ fn test_err_callx_unregistered() { mov64 r0, 0x2A exit", [], - TestContextObject::new(4), + TestContextObject::new(3), ProgramResult::Err(EbpfError::UnsupportedInstruction), ); } From e19fe44cf8a088b16a763b4496c4ef303fcc30a1 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 8 Nov 2024 19:27:02 -0300 Subject: [PATCH 05/13] Add test case --- tests/execution.rs | 67 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/tests/execution.rs b/tests/execution.rs index a814654a..8fc3d390 100644 --- a/tests/execution.rs +++ b/tests/execution.rs @@ -41,14 +41,31 @@ macro_rules! test_interpreter_and_jit { .unwrap(); }; ($executable:expr, $mem:tt, $context_object:expr, $expected_result:expr $(,)?) => { - test_interpreter_and_jit!(true, $executable, $mem, $context_object, $expected_result) + test_interpreter_and_jit!( + false, + true, + $executable, + $mem, + $context_object, + $expected_result + ) }; ($verify:literal, $executable:expr, $mem:tt, $context_object:expr, $expected_result:expr $(,)?) => { + test_interpreter_and_jit!( + false, + $verify, + $executable, + $mem, + $context_object, + $expected_result + ) + }; + ($override_budget:literal, $verify:literal, $executable:expr, $mem:tt, $context_object:expr, $expected_result:expr $(,)?) => { let expected_instruction_count = $context_object.get_remaining(); #[allow(unused_mut)] let mut context_object = $context_object; let expected_result = format!("{:?}", $expected_result); - if !expected_result.contains("ExceededMaxInstructions") { + if !$override_budget && !expected_result.contains("ExceededMaxInstructions") { context_object.remaining = INSTRUCTION_METER_BUDGET; } if $verify { @@ -3447,6 +3464,52 @@ fn test_invalid_exit_or_return() { } } +#[test] +fn callx_unsupported_instruction_and_exceeded_max_instructions() { + test_interpreter_and_jit_asm!( + " + sub32 r7, r1 + sub64 r5, 8 + sub64 r7, 0 + callx r5 + callx r5 + return + ", + [], + TestContextObject::new(4), + ProgramResult::Err(EbpfError::UnsupportedInstruction), + ); + + let prog = &[ + 0x1c, 0x17, 0x17, 0x95, 0x61, 0x61, 0x61, 0x61, // sub32 + 0x17, 0x55, 0x54, 0xff, 0x08, 0x00, 0x00, 0x00, // sub64 + 0x17, 0x17, 0x17, 0xff, 0x00, 0x00, 0x00, 0x00, // sub64 + 0x8d, 0x55, 0x54, 0xff, 0x09, 0x00, 0x00, 0x00, // callx + 0x8d, 0x55, 0x54, 0x23, 0x08, 0x00, 0x11, 0x4e, // callx + ]; + + let loader = Arc::new(BuiltinProgram::new_loader( + Config::default(), + FunctionRegistry::default(), + )); + let mut executable = Executable::::from_text_bytes( + prog, + loader, + SBPFVersion::V2, + FunctionRegistry::default(), + ) + .unwrap(); + + test_interpreter_and_jit!( + true, + false, + executable, + [], + TestContextObject::new(4), + ProgramResult::Err(EbpfError::UnsupportedInstruction) + ); +} + // SBPFv1 only [DEPRECATED] #[test] From a091e73f81e74fbc907a4ce1c6791f76caef7460 Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 8 Nov 2024 19:49:06 -0300 Subject: [PATCH 06/13] Test maximum instructions when callx is valid --- tests/execution.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/execution.rs b/tests/execution.rs index 8fc3d390..73c8033a 100644 --- a/tests/execution.rs +++ b/tests/execution.rs @@ -3510,6 +3510,23 @@ fn callx_unsupported_instruction_and_exceeded_max_instructions() { ); } +#[test] +fn test_maximum_after_callx() { + test_interpreter_and_jit_asm!( + " + mov64 r0, 0x0 + or64 r8, 0x20 + callx r8 + exit + function_foo: + mov64 r0, 0x2A + exit", + [], + TestContextObject::new(3), + ProgramResult::Err(EbpfError::ExceededMaxInstructions), + ); +} + // SBPFv1 only [DEPRECATED] #[test] From d991d17833a29da0befa111c47e20662972b4212 Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 12 Nov 2024 19:29:22 -0300 Subject: [PATCH 07/13] Rename test --- tests/execution.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/execution.rs b/tests/execution.rs index 73c8033a..0a4d2ced 100644 --- a/tests/execution.rs +++ b/tests/execution.rs @@ -3511,7 +3511,7 @@ fn callx_unsupported_instruction_and_exceeded_max_instructions() { } #[test] -fn test_maximum_after_callx() { +fn test_capped_after_callx() { test_interpreter_and_jit_asm!( " mov64 r0, 0x0 From 7ee20c0a89b78d6426e2befbe83f931fb500f12f Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 12 Nov 2024 19:38:02 -0300 Subject: [PATCH 08/13] Do not use assembly for tests --- tests/execution.rs | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/tests/execution.rs b/tests/execution.rs index 0a4d2ced..3cacd0dc 100644 --- a/tests/execution.rs +++ b/tests/execution.rs @@ -3466,40 +3466,27 @@ fn test_invalid_exit_or_return() { #[test] fn callx_unsupported_instruction_and_exceeded_max_instructions() { - test_interpreter_and_jit_asm!( - " + let program = " sub32 r7, r1 sub64 r5, 8 sub64 r7, 0 callx r5 callx r5 return - ", + "; + test_interpreter_and_jit_asm!( + program, [], TestContextObject::new(4), ProgramResult::Err(EbpfError::UnsupportedInstruction), ); - let prog = &[ - 0x1c, 0x17, 0x17, 0x95, 0x61, 0x61, 0x61, 0x61, // sub32 - 0x17, 0x55, 0x54, 0xff, 0x08, 0x00, 0x00, 0x00, // sub64 - 0x17, 0x17, 0x17, 0xff, 0x00, 0x00, 0x00, 0x00, // sub64 - 0x8d, 0x55, 0x54, 0xff, 0x09, 0x00, 0x00, 0x00, // callx - 0x8d, 0x55, 0x54, 0x23, 0x08, 0x00, 0x11, 0x4e, // callx - ]; - let loader = Arc::new(BuiltinProgram::new_loader( Config::default(), FunctionRegistry::default(), )); - let mut executable = Executable::::from_text_bytes( - prog, - loader, - SBPFVersion::V2, - FunctionRegistry::default(), - ) - .unwrap(); + let mut executable = assemble(program, loader).unwrap(); test_interpreter_and_jit!( true, false, From 08cac7fd75651dbe18545d36ae96556ba17bb485 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 18 Nov 2024 17:29:33 -0300 Subject: [PATCH 09/13] Test unregistered callx in both versions --- tests/execution.rs | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/tests/execution.rs b/tests/execution.rs index 3cacd0dc..23d6a9a4 100644 --- a/tests/execution.rs +++ b/tests/execution.rs @@ -2364,19 +2364,33 @@ fn test_callx() { #[test] fn test_err_callx_unregistered() { + let versions = [SBPFVersion::V1, SBPFVersion::V2]; + let expected_errors = [ + EbpfError::CallOutsideTextSegment, + EbpfError::UnsupportedInstruction, + ]; + // We execute three instructions when callx errors out. - test_interpreter_and_jit_asm!( - " - mov64 r0, 0x0 - or64 r8, 0x20 - callx r8 - exit - mov64 r0, 0x2A - exit", - [], - TestContextObject::new(3), - ProgramResult::Err(EbpfError::UnsupportedInstruction), - ); + for (version, error) in versions.iter().zip(expected_errors) { + let config = Config { + enabled_sbpf_versions: *version..=*version, + ..Config::default() + }; + + test_interpreter_and_jit_asm!( + " + mov64 r0, 0x0 + or64 r8, 0x20 + callx r8 + exit + mov64 r0, 0x2A + exit", + config, + [], + TestContextObject::new(3), + ProgramResult::Err(error), + ); + } } #[test] From deec3ad9e3ee677d69cce5b042d4bdbf874420a8 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 18 Nov 2024 18:49:01 -0300 Subject: [PATCH 10/13] Move logic to emit_undo_profile_instruction_count --- src/jit.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/jit.rs b/src/jit.rs index d4c1b25f..732d8fb1 100644 --- a/src/jit.rs +++ b/src/jit.rs @@ -979,9 +979,21 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { } #[inline] - fn emit_undo_profile_instruction_count(&mut self, target_pc: usize) { + fn emit_undo_profile_instruction_count(&mut self, target_pc: Option) { if self.config.enable_instruction_meter { - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, self.pc as i64 + 1 - target_pc as i64, None)); // instruction_meter += (self.pc + 1) - target_pc; + match target_pc { + Some(target_pc) => { + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, self.pc as i64 + 1 - target_pc as i64, None)); // instruction_meter += (self.pc + 1) - target_pc; + } + None => { + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x29, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter -= guest_target_pc + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, 1, None)); // instruction_meter += 1 + // Retrieve the current program from the stack. `return_near` popped an element from the stack, + // so the offset is 16. Check `ANCHOR_INTERNAL_FUNCTION_CALL_REG` for more details. + self.emit_ins(X86Instruction::load(OperandSize::S64, RSP, REGISTER_SCRATCH, X86IndirectAccess::OffsetIndexShift(-16, RSP, 0))); + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x01, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter += guest_current_pc + } + } } } @@ -1125,7 +1137,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { } } - self.emit_undo_profile_instruction_count(0); + self.emit_undo_profile_instruction_count(Some(0)); // Restore the previous frame pointer self.emit_ins(X86Instruction::pop(REGISTER_MAP[FRAME_PTR_REG])); @@ -1139,7 +1151,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_validate_and_profile_instruction_count(false, Some(0)); self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, function as usize as i64)); self.emit_ins(X86Instruction::call_immediate(self.relative_to_anchor(ANCHOR_EXTERNAL_FUNCTION_CALL, 5))); - self.emit_undo_profile_instruction_count(0); + self.emit_undo_profile_instruction_count(Some(0)); } #[inline] @@ -1224,7 +1236,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, target_pc as i64)); let jump_offset = self.relative_to_target_pc(target_pc, 6); self.emit_ins(X86Instruction::conditional_jump_immediate(op, jump_offset)); - self.emit_undo_profile_instruction_count(target_pc); + self.emit_undo_profile_instruction_count(Some(target_pc)); } #[inline] @@ -1245,7 +1257,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, target_pc as i64)); let jump_offset = self.relative_to_target_pc(target_pc, 6); self.emit_ins(X86Instruction::conditional_jump_immediate(op, jump_offset)); - self.emit_undo_profile_instruction_count(target_pc); + self.emit_undo_profile_instruction_count(Some(target_pc)); } fn emit_shift(&mut self, size: OperandSize, opcode_extension: u8, source: u8, destination: u8, immediate: Option) { @@ -1590,12 +1602,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { // If callx lands in an invalid address, we must undo the changes in the instruction meter // so that we can correctly calculate the number of executed instructions for error handling. self.set_anchor(ANCHOR_CALL_REG_UNSUPPORTED_INSTRUCTION); - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x29, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter -= guest_target_pc - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, 1, None)); // instruction_meter += 1 - // Retrieve the current program from the stack. `return_near` popped an element from the stack, - // so the offset is 16. - self.emit_ins(X86Instruction::load(OperandSize::S64, RSP, REGISTER_SCRATCH, X86IndirectAccess::OffsetIndexShift(-16, RSP, 0))); - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x01, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter += guest_current_pc + self.emit_undo_profile_instruction_count(None); self.emit_ins(X86Instruction::jump_immediate(self.relative_to_anchor(ANCHOR_CALL_UNSUPPORTED_INSTRUCTION, 5))); // Translates a vm memory address to a host memory address From ef72d90b942e7fb786382eb50ce4c75a76efd07d Mon Sep 17 00:00:00 2001 From: Lucas Date: Tue, 19 Nov 2024 11:37:47 -0300 Subject: [PATCH 11/13] Add more test cases --- tests/execution.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/execution.rs b/tests/execution.rs index 23d6a9a4..8b617501 100644 --- a/tests/execution.rs +++ b/tests/execution.rs @@ -2364,6 +2364,46 @@ fn test_callx() { #[test] fn test_err_callx_unregistered() { + let config = Config { + enabled_sbpf_versions: SBPFVersion::V1..=SBPFVersion::V1, + ..Config::default() + }; + + // Callx jumps to `mov64 r0, 0x2A` + test_interpreter_and_jit_asm!( + " + mov64 r0, 0x0 + lddw r8, 0x100000028 + callx r8 + exit + mov64 r0, 0x2A + exit", + config, + [], + TestContextObject::new(6), + ProgramResult::Ok(42), + ); + + let config = Config { + enabled_sbpf_versions: SBPFVersion::V2..=SBPFVersion::V2, + ..Config::default() + }; + + // Callx jumps to `mov64 r0, 0x2A` + test_interpreter_and_jit_asm!( + " + mov64 r0, 0x0 + or64 r8, 0x28 + callx r8 + exit + mov64 r0, 0x2A + exit", + config, + [], + TestContextObject::new(3), + ProgramResult::Err(EbpfError::UnsupportedInstruction), + ); + let versions = [SBPFVersion::V1, SBPFVersion::V2]; let expected_errors = [ EbpfError::CallOutsideTextSegment, @@ -2377,6 +2417,7 @@ fn test_err_callx_unregistered() { ..Config::default() }; + // Callx jumps to a location outside text segment test_interpreter_and_jit_asm!( " mov64 r0, 0x0 From 4e2acc4ef880790b559168e35df36940ea078df9 Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 21 Nov 2024 14:57:43 -0300 Subject: [PATCH 12/13] Refactor emit_unfo_profile_instruction_count --- src/jit.rs | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/jit.rs b/src/jit.rs index 732d8fb1..351799cd 100644 --- a/src/jit.rs +++ b/src/jit.rs @@ -979,20 +979,19 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { } #[inline] - fn emit_undo_profile_instruction_count(&mut self, target_pc: Option) { + fn emit_undo_profile_instruction_count(&mut self, target_pc: Value) { if self.config.enable_instruction_meter { match target_pc { - Some(target_pc) => { - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, self.pc as i64 + 1 - target_pc as i64, None)); // instruction_meter += (self.pc + 1) - target_pc; + Value::Constant64(target_pc, _) => { + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, self.pc as i64 + 1 - target_pc, None)); // instruction_meter += (self.pc + 1) - target_pc; } - None => { - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x29, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter -= guest_target_pc + Value::Register(reg) => { + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x29, reg, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter -= guest_target_pc self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, 1, None)); // instruction_meter += 1 - // Retrieve the current program from the stack. `return_near` popped an element from the stack, - // so the offset is 16. Check `ANCHOR_INTERNAL_FUNCTION_CALL_REG` for more details. self.emit_ins(X86Instruction::load(OperandSize::S64, RSP, REGISTER_SCRATCH, X86IndirectAccess::OffsetIndexShift(-16, RSP, 0))); self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x01, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter += guest_current_pc } + _ => {}, } } } @@ -1137,7 +1136,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { } } - self.emit_undo_profile_instruction_count(Some(0)); + self.emit_undo_profile_instruction_count(Value::Constant64(0, false)); // Restore the previous frame pointer self.emit_ins(X86Instruction::pop(REGISTER_MAP[FRAME_PTR_REG])); @@ -1151,7 +1150,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_validate_and_profile_instruction_count(false, Some(0)); self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, function as usize as i64)); self.emit_ins(X86Instruction::call_immediate(self.relative_to_anchor(ANCHOR_EXTERNAL_FUNCTION_CALL, 5))); - self.emit_undo_profile_instruction_count(Some(0)); + self.emit_undo_profile_instruction_count(Value::Constant64(0, false)); } #[inline] @@ -1236,7 +1235,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, target_pc as i64)); let jump_offset = self.relative_to_target_pc(target_pc, 6); self.emit_ins(X86Instruction::conditional_jump_immediate(op, jump_offset)); - self.emit_undo_profile_instruction_count(Some(target_pc)); + self.emit_undo_profile_instruction_count(Value::Constant64(target_pc as i64, false)); } #[inline] @@ -1257,7 +1256,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, target_pc as i64)); let jump_offset = self.relative_to_target_pc(target_pc, 6); self.emit_ins(X86Instruction::conditional_jump_immediate(op, jump_offset)); - self.emit_undo_profile_instruction_count(Some(target_pc)); + self.emit_undo_profile_instruction_count(Value::Constant64(target_pc as i64, false)); } fn emit_shift(&mut self, size: OperandSize, opcode_extension: u8, source: u8, destination: u8, immediate: Option) { @@ -1602,7 +1601,11 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { // If callx lands in an invalid address, we must undo the changes in the instruction meter // so that we can correctly calculate the number of executed instructions for error handling. self.set_anchor(ANCHOR_CALL_REG_UNSUPPORTED_INSTRUCTION); - self.emit_undo_profile_instruction_count(None); + self.emit_ins(X86Instruction::mov(OperandSize::S64, REGISTER_SCRATCH, REGISTER_MAP[0])); + // Retrieve the current program from the stack. `return_near` popped an element from the stack, + // so the offset is 16. Check `ANCHOR_INTERNAL_FUNCTION_CALL_REG` for more details. + self.emit_ins(X86Instruction::load(OperandSize::S64, RSP, REGISTER_SCRATCH, X86IndirectAccess::OffsetIndexShift(-16, RSP, 0))); + self.emit_undo_profile_instruction_count(Value::Register(REGISTER_MAP[0])); self.emit_ins(X86Instruction::jump_immediate(self.relative_to_anchor(ANCHOR_CALL_UNSUPPORTED_INSTRUCTION, 5))); // Translates a vm memory address to a host memory address From 2959efa0175731ccf2e7474a15e2e98a7c927b6e Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 21 Nov 2024 16:13:23 -0300 Subject: [PATCH 13/13] Fix comments --- src/jit.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/jit.rs b/src/jit.rs index 351799cd..c0691196 100644 --- a/src/jit.rs +++ b/src/jit.rs @@ -985,13 +985,12 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { Value::Constant64(target_pc, _) => { self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, self.pc as i64 + 1 - target_pc, None)); // instruction_meter += (self.pc + 1) - target_pc; } - Value::Register(reg) => { - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x29, reg, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter -= guest_target_pc + Value::Register(target_pc) => { + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x29, target_pc, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter -= guest_target_pc self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x81, 0, REGISTER_INSTRUCTION_METER, 1, None)); // instruction_meter += 1 - self.emit_ins(X86Instruction::load(OperandSize::S64, RSP, REGISTER_SCRATCH, X86IndirectAccess::OffsetIndexShift(-16, RSP, 0))); - self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x01, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter += guest_current_pc + self.emit_ins(X86Instruction::alu(OperandSize::S64, 0x01, REGISTER_SCRATCH, REGISTER_INSTRUCTION_METER, 0, None)); // instruction_meter += self.pc } - _ => {}, + _ => debug_assert!(false), } } } @@ -1235,7 +1234,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, target_pc as i64)); let jump_offset = self.relative_to_target_pc(target_pc, 6); self.emit_ins(X86Instruction::conditional_jump_immediate(op, jump_offset)); - self.emit_undo_profile_instruction_count(Value::Constant64(target_pc as i64, false)); + self.emit_undo_profile_instruction_count(Value::Constant64(target_pc as i64, true)); } #[inline] @@ -1256,7 +1255,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { self.emit_ins(X86Instruction::load_immediate(OperandSize::S64, REGISTER_SCRATCH, target_pc as i64)); let jump_offset = self.relative_to_target_pc(target_pc, 6); self.emit_ins(X86Instruction::conditional_jump_immediate(op, jump_offset)); - self.emit_undo_profile_instruction_count(Value::Constant64(target_pc as i64, false)); + self.emit_undo_profile_instruction_count(Value::Constant64(target_pc as i64, true)); } fn emit_shift(&mut self, size: OperandSize, opcode_extension: u8, source: u8, destination: u8, immediate: Option) { @@ -1602,7 +1601,7 @@ impl<'a, C: ContextObject> JitCompiler<'a, C> { // so that we can correctly calculate the number of executed instructions for error handling. self.set_anchor(ANCHOR_CALL_REG_UNSUPPORTED_INSTRUCTION); self.emit_ins(X86Instruction::mov(OperandSize::S64, REGISTER_SCRATCH, REGISTER_MAP[0])); - // Retrieve the current program from the stack. `return_near` popped an element from the stack, + // Retrieve the current program counter from the stack. `return_near` popped an element from the stack, // so the offset is 16. Check `ANCHOR_INTERNAL_FUNCTION_CALL_REG` for more details. self.emit_ins(X86Instruction::load(OperandSize::S64, RSP, REGISTER_SCRATCH, X86IndirectAccess::OffsetIndexShift(-16, RSP, 0))); self.emit_undo_profile_instruction_count(Value::Register(REGISTER_MAP[0]));