diff --git a/Cargo.lock b/Cargo.lock index d67141996827b..e1c980d98de11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3463,6 +3463,7 @@ dependencies = [ "rustc_macros", "rustc_metadata", "rustc_middle", + "rustc_mir_transform", "rustc_monomorphize", "rustc_query_system", "rustc_serialize", diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 3ab4cd0a0f53c..4b093085c9278 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -27,6 +27,7 @@ rustc_index = { path = "../rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_metadata = { path = "../rustc_metadata" } rustc_middle = { path = "../rustc_middle" } +rustc_mir_transform = { path = "../rustc_mir_transform" } rustc_monomorphize = { path = "../rustc_monomorphize" } rustc_query_system = { path = "../rustc_query_system" } rustc_serialize = { path = "../rustc_serialize" } diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 386e1f91e7f0c..26e14610ac737 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -14,16 +14,15 @@ use super::FunctionCx; use crate::traits::*; pub(crate) fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - fx: &FunctionCx<'a, 'tcx, Bx>, + fx: &FunctionCx<'a, '_, 'tcx, Bx>, ) -> BitSet { - let mir = fx.mir; + let mir = &fx.mir; let dominators = mir.basic_blocks.dominators(); let locals = mir .local_decls .iter() .map(|decl| { - let ty = fx.monomorphize(decl.ty); - let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span); + let layout = fx.cx.spanned_layout_of(decl.ty, decl.source_info.span); if layout.is_zst() { LocalKind::ZST } else if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) { @@ -70,7 +69,7 @@ enum LocalKind { } struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { - fx: &'mir FunctionCx<'a, 'tcx, Bx>, + fx: &'mir FunctionCx<'a, 'mir, 'tcx, Bx>, dominators: &'mir Dominators, locals: IndexVec, } @@ -110,10 +109,9 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, ); if is_consume { let base_ty = place_base.ty(self.fx.mir, cx.tcx()); - let base_ty = self.fx.monomorphize(base_ty); // ZSTs don't require any actual memory access. - let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty; + let elem_ty = base_ty.projection_ty(cx.tcx(), elem).ty; let span = self.fx.mir.local_decls[place_ref.local].source_info.span; if cx.spanned_layout_of(elem_ty, span).is_zst() { return; @@ -237,7 +235,6 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx> let kind = &mut self.locals[local]; if *kind != LocalKind::Memory { let ty = self.fx.mir.local_decls[local].ty; - let ty = self.fx.monomorphize(ty); if self.fx.cx.type_needs_drop(ty) { // Only need the place if we're actually dropping it. *kind = LocalKind::Memory; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 817e2ca72ec12..b5134ce9331c4 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -37,17 +37,17 @@ enum MergingSucc { /// Used by `FunctionCx::codegen_terminator` for emitting common patterns /// e.g., creating a basic block, calling a function, etc. -struct TerminatorCodegenHelper<'tcx> { +struct TerminatorCodegenHelper<'mir, 'tcx> { bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, + terminator: &'mir mir::Terminator<'tcx>, } -impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { +impl<'a, 'tcx> TerminatorCodegenHelper<'_, 'tcx> { /// Returns the appropriate `Funclet` for the current funclet, if on MSVC, /// either already previously cached, or newly created, by `landing_pad_for`. fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>( &self, - fx: &'b mut FunctionCx<'a, 'tcx, Bx>, + fx: &'b mut FunctionCx<'a, '_, 'tcx, Bx>, ) -> Option<&'b Bx::Funclet> { let cleanup_kinds = fx.cleanup_kinds.as_ref()?; let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?; @@ -73,7 +73,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// stuff in it or next to it. fn llbb_with_cleanup>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'a, '_, 'tcx, Bx>, target: mir::BasicBlock, ) -> Bx::BasicBlock { let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target); @@ -97,7 +97,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { fn llbb_characteristics>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'a, '_, 'tcx, Bx>, target: mir::BasicBlock, ) -> (bool, bool) { if let Some(ref cleanup_kinds) = fx.cleanup_kinds { @@ -122,7 +122,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { fn funclet_br>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'a, '_, 'tcx, Bx>, bx: &mut Bx, target: mir::BasicBlock, mergeable_succ: bool, @@ -151,7 +151,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// return destination `destination` and the unwind action `unwind`. fn do_call>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'a, '_, 'tcx, Bx>, bx: &mut Bx, fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, fn_ptr: Bx::Value, @@ -270,7 +270,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// Generates inline assembly with optional `destination` and `unwind`. fn do_inlineasm>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'a, '_, 'tcx, Bx>, bx: &mut Bx, template: &[InlineAsmTemplatePiece], operands: &[InlineAsmOperandRef<'tcx, Bx>], @@ -337,9 +337,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } /// Codegen implementations for some terminator variants. -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { /// Generates code for a `Resume` terminator. - fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) { + fn codegen_resume_terminator( + &mut self, + helper: TerminatorCodegenHelper<'_, 'tcx>, + bx: &mut Bx, + ) { if let Some(funclet) = helper.funclet(self) { bx.cleanup_ret(funclet, None); } else { @@ -356,7 +360,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_switchint_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, discr: &mir::Operand<'tcx>, targets: &SwitchTargets, @@ -496,7 +500,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[tracing::instrument(level = "trace", skip(self, helper, bx))] fn codegen_drop_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, source_info: &mir::SourceInfo, location: mir::Place<'tcx>, @@ -505,7 +509,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mergeable_succ: bool, ) -> MergingSucc { let ty = location.ty(self.mir, bx.tcx()).ty; - let ty = self.monomorphize(ty); let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty); if let ty::InstanceKind::DropGlue(_, None) = drop_fn.def { @@ -638,7 +641,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_assert_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, cond: &mir::Operand<'tcx>, @@ -717,7 +720,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_terminate_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, reason: UnwindTerminateReason, @@ -747,7 +750,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { /// Returns `Some` if this is indeed a panic intrinsic and codegen is done. fn codegen_panic_intrinsic( &mut self, - helper: &TerminatorCodegenHelper<'tcx>, + helper: &TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, intrinsic: ty::IntrinsicDef, instance: Option>, @@ -815,7 +818,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_call_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, func: &mir::Operand<'tcx>, @@ -868,10 +871,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let abi = sig.abi(); let extra_args = &args[sig.inputs().skip_binder().len()..]; - let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| { - let op_ty = op_arg.node.ty(self.mir, bx.tcx()); - self.monomorphize(op_ty) - })); + let extra_args = bx.tcx().mk_type_list_from_iter( + extra_args.iter().map(|op_arg| op_arg.node.ty(self.mir, bx.tcx())), + ); let fn_abi = match instance { Some(instance) => bx.fn_abi_of_instance(instance, extra_args), @@ -1154,7 +1156,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_asm_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'_, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, template: &[ast::InlineAsmTemplatePiece], @@ -1196,8 +1198,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { InlineAsmOperandRef::Const { string } } mir::InlineAsmOperand::SymFn { ref value } => { - let const_ = self.monomorphize(value.const_); - if let ty::FnDef(def_id, args) = *const_.ty().kind() { + if let ty::FnDef(def_id, args) = *value.const_.ty().kind() { let instance = ty::Instance::resolve_for_fn_ptr( bx.tcx(), ty::ParamEnv::reveal_all(), @@ -1238,7 +1239,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { pub fn codegen_block(&mut self, mut bb: mir::BasicBlock) { let llbb = match self.try_llbb(bb) { Some(llbb) => llbb, @@ -1293,7 +1294,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &mut self, bx: &mut Bx, bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, + terminator: &mir::Terminator<'tcx>, ) -> MergingSucc { debug!("codegen_terminator: {:?}", terminator); diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index 0aa85b8203825..3d79a7ca81dd7 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -9,21 +9,21 @@ use crate::errors; use crate::mir::operand::OperandRef; use crate::traits::*; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { pub fn eval_mir_constant_to_operand( &self, bx: &mut Bx, constant: &mir::ConstOperand<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { let val = self.eval_mir_constant(constant); - let ty = self.monomorphize(constant.ty()); - OperandRef::from_const(bx, val, ty) + OperandRef::from_const(bx, val, constant.ty()) } pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue<'tcx> { // `MirUsedCollector` visited all required_consts before codegen began, so if we got here // there can be no more constants that fail to evaluate. - self.monomorphize(constant.const_) + constant + .const_ .eval(self.cx.tcx(), ty::ParamEnv::reveal_all(), constant.span) .expect("erroneous constant missed by mono item collection") } @@ -37,7 +37,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &self, constant: &mir::ConstOperand<'tcx>, ) -> Result, Ty<'tcx>>, ErrorHandled> { - let uv = match self.monomorphize(constant.const_) { + let uv = match constant.const_ { mir::Const::Unevaluated(uv, _) => uv.shrink(), mir::Const::Ty(_, c) => match c.kind() { // A constant that came from a const generic but was then used as an argument to old-style @@ -55,7 +55,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // const generic, and get rid of this entire function. other => span_bug!(constant.span, "{other:#?}"), }; - let uv = self.monomorphize(uv); self.cx.tcx().const_eval_resolve_for_typeck(ty::ParamEnv::reveal_all(), uv, constant.span) } @@ -65,7 +64,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &Bx, constant: &mir::ConstOperand<'tcx>, ) -> (Bx::Value, Ty<'tcx>) { - let ty = self.monomorphize(constant.ty()); + let ty = constant.ty(); let ty_is_simd = ty.is_simd(); // FIXME: ideally we'd assert that this is a SIMD type, but simd_shuffle // in its current form relies on a regular array being passed as an diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs index 67f1ef5d9449c..4e8c8e7c338b7 100644 --- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs @@ -4,14 +4,10 @@ use rustc_middle::mir::SourceScope; use super::FunctionCx; use crate::traits::*; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { pub fn codegen_coverage(&self, bx: &mut Bx, kind: &CoverageKind, scope: SourceScope) { // Determine the instance that coverage data was originally generated for. - let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) { - self.monomorphize(inlined) - } else { - self.instance - }; + let instance = scope.inlined_instance(&self.mir.source_scopes).unwrap_or(self.instance); // Handle the coverage info in a backend-specific way. bx.add_coverage(instance, kind); diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 75692540c0345..70d7473670cf8 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -201,7 +201,7 @@ fn calculate_debuginfo_offset< DebugInfoOffset { direct_offset, indirect_offsets, result: place } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) { bx.set_span(source_info.span); if let Some(dbg_loc) = self.dbg_loc(source_info) { @@ -279,9 +279,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // FIXME(eddyb) is this `+ 1` needed at all? let kind = VariableKind::ArgumentVariable(arg_index + 1); - let arg_ty = self.monomorphize(decl.ty); - - self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span) + self.cx.create_dbg_var(name, decl.ty, dbg_scope, kind, span) }, ) } else { @@ -457,13 +455,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let var_ty = if let Some(ref fragment) = var.composite { - self.monomorphize(fragment.ty) + fragment.ty } else { match var.value { mir::VarDebugInfoContents::Place(place) => { self.monomorphized_place_ty(place.as_ref()) } - mir::VarDebugInfoContents::Const(c) => self.monomorphize(c.ty()), + mir::VarDebugInfoContents::Const(c) => c.ty(), } }; diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 4acbc04c505e8..361cce34aa376 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -49,7 +49,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx.memset(dst, val, size, align, flags); } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { /// In the `Err` case, returns the instance that should be called instead. pub fn codegen_intrinsic_call( bx: &mut Bx, diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs index 93f0ab36f2a2f..36129f42056f9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/locals.rs +++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs @@ -37,7 +37,7 @@ impl<'tcx, V> Locals<'tcx, V> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { pub(super) fn initialize_locals(&mut self, values: Vec>) { assert!(self.locals.values.is_empty()); // FIXME(#115215): After #115025 get's merged this might not be necessary @@ -46,7 +46,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { LocalRef::Place(_) | LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => (), LocalRef::Operand(op) => { let local = mir::Local::from_usize(local); - let expected_ty = self.monomorphize(self.mir.local_decls[local].ty); + let expected_ty = self.mir.local_decls[local].ty; if expected_ty != op.layout.ty { warn!( "Unexpected initial operand type:\nexpected {expected_ty:?},\nfound {:?}.\n\ @@ -68,7 +68,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { match value { LocalRef::Place(_) | LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => (), LocalRef::Operand(ref mut op) => { - let local_ty = self.monomorphize(self.mir.local_decls[local].ty); + let local_ty = self.mir.local_decls[local].ty; if local_ty != op.layout.ty { // FIXME(#112651): This can be changed to an ICE afterwards. debug!("updating type of operand due to subtyping"); diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index de94d87bcea7a..8824cb504b4f2 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -5,8 +5,9 @@ use rustc_index::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::{traversal, UnwindTerminateReason}; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout}; -use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt}; +use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt}; use rustc_middle::{bug, mir, span_bug}; +use rustc_mir_transform::{add_call_guards, dump_mir, pass_manager}; use rustc_target::abi::call::{FnAbi, PassMode}; use tracing::{debug, instrument}; @@ -42,10 +43,11 @@ enum CachedLlbb { } /// Master context for codegenning from MIR. -pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { +pub struct FunctionCx<'a, 'mir, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { instance: Instance<'tcx>, - mir: &'tcx mir::Body<'tcx>, + /// Store the monomorphized MIR body directly. + mir: &'mir mir::Body<'tcx>, debug_context: Option>, @@ -114,20 +116,6 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { caller_location: Option>, } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { - pub fn monomorphize(&self, value: T) -> T - where - T: Copy + TypeFoldable>, - { - debug!("monomorphize: self.instance={:?}", self.instance); - self.instance.instantiate_mir_and_normalize_erasing_regions( - self.cx.tcx(), - ty::ParamEnv::reveal_all(), - ty::EarlyBinder::bind(value), - ) - } -} - enum LocalRef<'tcx, V> { Place(PlaceRef<'tcx, V>), /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place). @@ -165,7 +153,23 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let llfn = cx.get_fn(instance); - let mir = cx.tcx().instance_mir(instance.def); + let mut mir = instance.instantiate_mir_and_normalize_erasing_regions( + cx.tcx(), + ty::ParamEnv::reveal_all(), + ty::EarlyBinder::bind(cx.tcx().instance_mir(instance.def).clone()), + ); + pass_manager::run_passes( + cx.tcx(), + &mut mir, + &[ + // Some cleanup necessary at least for LLVM and potentially other codegen backends. + &add_call_guards::CriticalCallEdges, + // Dump the end result for testing and debugging purposes. + &dump_mir::Marker("Monomorphic"), + ], + Some(mir::MirPhase::Runtime(mir::RuntimePhase::Monomorphic)), + ); + let mir = &mir; let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); debug!("fn_abi: {:?}", fn_abi); @@ -226,7 +230,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let mut allocate_local = |local| { let decl = &mir.local_decls[local]; - let layout = start_bx.layout_of(fx.monomorphize(decl.ty)); + let layout = start_bx.layout_of(decl.ty); assert!(!layout.ty.has_erasable_regions()); if local == mir::RETURN_PLACE { @@ -296,7 +300,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /// indirect. fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'a, '_, 'tcx, Bx>, memory_locals: &BitSet, ) -> Vec> { let mir = fx.mir; @@ -316,7 +320,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( .enumerate() .map(|(arg_index, local)| { let arg_decl = &mir.local_decls[local]; - let arg_ty = fx.monomorphize(arg_decl.ty); + let arg_ty = arg_decl.ty; if Some(local) == mir.spread_arg { // This argument (e.g., the last argument in the "rust-call" ABI) diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 1891de8c0eb37..a242d35c7780d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -549,7 +549,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, @@ -637,7 +637,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } mir::Operand::Constant(ref constant) => { - let constant_ty = self.monomorphize(constant.ty()); + let constant_ty = constant.ty(); // Most SIMD vector constants should be passed as immediates. // (In particular, some intrinsics really rely on this.) if constant_ty.is_simd() { diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 0fad4d169edd5..71c829dbbbbbf 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -453,7 +453,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] pub fn codegen_place( &mut self, @@ -497,7 +497,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::ProjectionElem::OpaqueCast(ty) => { bug!("encountered OpaqueCast({ty}) in codegen") } - mir::ProjectionElem::Subtype(ty) => cg_base.project_type(bx, self.monomorphize(ty)), + mir::ProjectionElem::Subtype(ty) => cg_base.project_type(bx, ty), mir::ProjectionElem::Index(index) => { let index = &mir::Operand::Copy(mir::Place::from(index)); let index = self.codegen_operand(bx, index); @@ -518,7 +518,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from)); let projected_ty = PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty; - subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); + subslice.layout = bx.cx().layout_of(projected_ty); if subslice.layout.is_unsized() { assert!(from_end, "slice subslices should be `from_end`"); @@ -539,7 +539,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> { let tcx = self.cx.tcx(); let place_ty = place_ref.ty(self.mir, tcx); - self.monomorphize(place_ty.ty) + place_ty.ty } } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 91fd9905f63cc..7776797b0b703 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -17,7 +17,7 @@ use crate::common::IntPredicate; use crate::traits::*; use crate::{base, MemFlags}; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] pub fn codegen_rvalue( &mut self, @@ -112,9 +112,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - let count = self - .monomorphize(count) - .eval_target_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); + let count = count.eval_target_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all()); bx.write_operand_repeatedly(cg_elem, count, dest); } @@ -454,7 +452,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => { let operand = self.codegen_operand(bx, source); debug!("cast operand is {:?}", operand); - let cast = bx.cx().layout_of(self.monomorphize(mir_cast_ty)); + let cast = bx.cx().layout_of(mir_cast_ty); let val = match *kind { mir::CastKind::PointerExposeProvenance => { @@ -679,7 +677,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Discriminant(ref place) => { let discr_ty = rvalue.ty(self.mir, bx.tcx()); - let discr_ty = self.monomorphize(discr_ty); let discr = self.codegen_place(bx, place.as_ref()).codegen_get_discr(bx, discr_ty); OperandRef { val: OperandValue::Immediate(discr), @@ -688,7 +685,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } mir::Rvalue::NullaryOp(ref null_op, ty) => { - let ty = self.monomorphize(ty); let layout = bx.cx().layout_of(ty); let val = match null_op { mir::NullOp::SizeOf => { @@ -747,7 +743,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::Rvalue::Repeat(..) => bug!("{rvalue:?} in codegen_rvalue_operand"), mir::Rvalue::Aggregate(_, ref fields) => { let ty = rvalue.ty(self.mir, self.cx.tcx()); - let ty = self.monomorphize(ty); let layout = self.cx.layout_of(ty); // `rvalue_creates_operand` has arranged that we only get here if @@ -789,7 +784,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let operand = self.codegen_operand(bx, operand); let val = operand.immediate(); - let content_ty = self.monomorphize(content_ty); let box_layout = bx.cx().layout_of(Ty::new_box(bx.tcx(), content_ty)); OperandRef { val: OperandValue::Immediate(val), layout: box_layout } @@ -1049,13 +1043,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool { match *rvalue { mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, cast_ty) => { let operand_ty = operand.ty(self.mir, self.cx.tcx()); - let cast_layout = self.cx.layout_of(self.monomorphize(cast_ty)); - let operand_layout = self.cx.layout_of(self.monomorphize(operand_ty)); + let cast_layout = self.cx.layout_of(cast_ty); + let operand_layout = self.cx.layout_of(operand_ty); match (self.value_kind(operand_layout), self.value_kind(cast_layout)) { // Can always load from a pointer as needed @@ -1114,8 +1108,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false, }; allowed_kind && { - let ty = rvalue.ty(self.mir, self.cx.tcx()); - let ty = self.monomorphize(ty); + let ty = rvalue.ty(self.mir, self.cx.tcx()); let layout = self.cx.spanned_layout_of(ty, span); !self.cx.is_backend_ref(layout) } diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 73283cafb4947..e19f6f70f5d63 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -5,7 +5,7 @@ use tracing::instrument; use super::{FunctionCx, LocalRef}; use crate::traits::*; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, '_, 'tcx, Bx> { #[instrument(level = "debug", skip(self, bx))] pub fn codegen_statement(&mut self, bx: &mut Bx, statement: &mir::Statement<'tcx>) { self.set_debug_loc(bx, statement.source_info); diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 081a23b6ff317..b12e083e13b39 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -159,6 +159,7 @@ impl RuntimePhase { "initial" => Self::Initial, "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup, "optimized" => Self::Optimized, + "monomorphic" => Self::Monomorphic, _ => bug!("Unknown runtime phase: '{}'", phase), } } diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 22a4b688c517c..2dd1e2a0f6512 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -99,6 +99,7 @@ impl MirPhase { MirPhase::Runtime(RuntimePhase::Initial) => "runtime", MirPhase::Runtime(RuntimePhase::PostCleanup) => "runtime-post-cleanup", MirPhase::Runtime(RuntimePhase::Optimized) => "runtime-optimized", + MirPhase::Runtime(RuntimePhase::Monomorphic) => "runtime-monomorphic", } } @@ -153,6 +154,8 @@ pub enum RuntimePhase { /// * [`ProjectionElem::Deref`] of `Box` PostCleanup = 1, Optimized = 2, + /// This corresponds to monomorphised MIR that is being processed by codegen. + Monomorphic = 3, } /////////////////////////////////////////////////////////////////////////// diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 62e73ba2c8e24..f075c2075d093 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -38,12 +38,12 @@ use rustc_trait_selection::traits; use tracing::{debug, trace}; #[macro_use] -mod pass_manager; +pub mod pass_manager; use pass_manager::{self as pm, Lint, MirLint, MirPass, WithMinOptLevel}; mod abort_unwinding_calls; -mod add_call_guards; +pub mod add_call_guards; mod add_moves_for_packed_drops; mod add_retag; mod add_subtyping_projections; @@ -606,8 +606,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &multiple_return_terminators::MultipleReturnTerminators, &deduplicate_blocks::DeduplicateBlocks, &large_enums::EnumSizeOpt { discrepancy: 128 }, - // Some cleanup necessary at least for LLVM and potentially other codegen backends. - &add_call_guards::CriticalCallEdges, // Cleanup for human readability, off by default. &prettify::ReorderBasicBlocks, &prettify::ReorderLocals, diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 28d4e1a1c9184..3e14ead28ab84 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -59,7 +59,7 @@ const fn c_name(name: &'static str) -> &'static str { /// A streamlined trait that you can implement to create a pass; the /// pass will be named after the type, and it will consist of a main /// loop that goes over each available MIR and applies `run_pass`. -pub(super) trait MirPass<'tcx> { +pub trait MirPass<'tcx> { fn name(&self) -> &'static str { // FIXME Simplify the implementation once more `str` methods get const-stable. // See copypaste in `MirLint` @@ -160,7 +160,7 @@ pub(super) fn run_passes_no_validate<'tcx>( } /// The optional `phase_change` is applied after executing all the passes, if present -pub(super) fn run_passes<'tcx>( +pub fn run_passes<'tcx>( tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>], diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 69e2592e82c77..b47aae16513b9 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -391,7 +391,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { // is used to simplify inserting code that should be executed along the return edge // from the call. FIXME(tmiasko): Since this is a strictly code generation concern, // the code generation should be responsible for handling it. - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized) + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Monomorphic) && self.is_critical_call_edge(target, unwind) { self.fail( diff --git a/tests/ui/mir/validate/critical-edge.rs b/tests/ui/mir/validate/critical-edge.rs index 9048d08a22a73..d9c37208e549b 100644 --- a/tests/ui/mir/validate/critical-edge.rs +++ b/tests/ui/mir/validate/critical-edge.rs @@ -2,15 +2,14 @@ // //@ build-fail //@ edition: 2021 -//@ compile-flags: --crate-type=lib +//@ compile-flags: --crate-type=lib -Zvalidate-mir //@ failure-status: 101 //@ dont-check-compiler-stderr //@ error-pattern: encountered critical edge in `Call` terminator #![feature(custom_mir, core_intrinsics)] use core::intrinsics::mir::*; -#[custom_mir(dialect = "runtime", phase = "optimized")] -#[inline(always)] +#[custom_mir(dialect = "runtime", phase = "monomorphic")] pub fn f(a: u32) -> u32 { mir! { {