diff --git a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs index b4f54f4c79f51..fe48272f278c8 100644 --- a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs @@ -98,6 +98,7 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine { _ecx: &mut InterpCx<'tcx, Self>, _instance: ty::Instance<'tcx>, _args: &[interpret::OpTy<'tcx, Self::Provenance>], + _caller_moved_locals: &mut Vec, _destination: &interpret::PlaceTy<'tcx, Self::Provenance>, _target: Option, _unwind: UnwindAction, diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 7c10dd04f39f3..78724c81e3731 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -470,6 +470,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> { ecx: &mut InterpCx<'tcx, Self>, instance: ty::Instance<'tcx>, args: &[OpTy<'tcx>], + _caller_moved_locals: &mut Vec, dest: &PlaceTy<'tcx, Self::Provenance>, target: Option, _unwind: mir::UnwindAction, @@ -879,8 +880,10 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> { ) -> InterpResult<'tcx> { use rustc_middle::mir::AssertKind::*; // Convert `AssertKind` to `AssertKind`. - let eval_to_int = - |op| ecx.read_immediate(&ecx.eval_operand(op, None)?).map(|x| x.to_const_int()); + let mut eval_to_int = |op| { + let op = ecx.eval_operand(op, None)?; + ecx.read_immediate(&op).map(|x| x.to_const_int()) + }; let err = match msg { BoundsCheck { len, index } => { let len = eval_to_int(len)?; diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index c378a70da4b2b..d9d6c827bd736 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -1,8 +1,8 @@ //! Manages calling a concrete function (with known MIR body) with argument passing, //! and returning the return value to the caller. -use std::assert_matches; use std::borrow::Cow; +use std::{assert_matches, debug_assert_matches}; use either::{Left, Right}; use rustc_abi::{self as abi, ExternAbi, FieldIdx, Integer, VariantIdx}; @@ -385,8 +385,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.storage_live_dyn(local, meta)?; } // Now we can finally actually evaluate the callee place. - let callee_arg = - self.eval_place(*callee_arg, /* skip_validity_for_simple_deref */ false)?; + let callee_arg = self + .eval_place_for_write(*callee_arg, /* skip_validity_for_simple_deref */ false)?; // We allow some transmutes here. // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This // is true for all `copy_op`, but there are a lot of special cases for argument passing @@ -550,6 +550,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let (callee_arg_idx, callee_abi) = callee_args_abis.next().unwrap(); assert!(callee_abi.layout.is_1zst() && callee_abi.is_ignore()); ecx.storage_live(local)?; + ecx.allocate_local_for_write(local)?; // And skip it in the caller, if present. We can tell whether it is present by // comparing the number of arguments on the caller and callee side. if caller_fn_abi.args.len() == callee_fn_abi.args.len() { @@ -569,8 +570,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // This argument is a VaList holding the remaining caller-side arguments. ecx.storage_live(local)?; - let place = - ecx.eval_place(dest, /* skip_validity_for_simple_deref */ false)?; + let place = ecx.eval_place_for_write( + dest, /* skip_validity_for_simple_deref */ false, + )?; let mplace = ecx.force_allocation(&place)?; // Consume the remaining arguments by putting them into the variable argument @@ -596,6 +598,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } else if Some(local) == body.spread_arg { // Make the local live once, then fill in the value field by field. ecx.storage_live(local)?; + // Function arguments start allocated, including an empty spread tuple for + // which the loop below has no fields to initialize. + ecx.allocate_local_for_write(local)?; // Must be a tuple let ty::Tuple(fields) = ty.kind() else { span_bug!(ecx.cur_span(), "non-tuple type for `spread_arg`: {ty}") @@ -673,6 +678,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn_val: FnVal<'tcx, M::ExtraFnVal>, (caller_abi, caller_fn_abi): (ExternAbi, Option<&FnAbi<'tcx, Ty<'tcx>>>), args: &[FnArg<'tcx, M::Provenance>], + mut caller_moved_locals: Vec, with_caller_location: bool, destination: &PlaceTy<'tcx, M::Provenance>, target: Option, @@ -687,15 +693,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { FnVal::Other(extra) => { let caller_fn_abi = caller_fn_abi.expect("FnAbi should have been computed for this call"); - return M::call_extra_fn( - self, - extra, - caller_fn_abi, - args, - destination, - target, - unwind, - ); + M::call_extra_fn(self, extra, caller_fn_abi, args, destination, target, unwind)?; + return self.deallocate_moved_locals(caller_moved_locals); } }; @@ -707,6 +706,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self, instance, &Self::copy_fn_args(args), + &mut caller_moved_locals, destination, target, unwind, @@ -717,13 +717,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { FnVal::Instance(fallback), (caller_abi, caller_fn_abi), args, + caller_moved_locals, with_caller_location, destination, target, unwind, ); } else { - interp_ok(()) + self.deallocate_moved_locals(caller_moved_locals) } } ty::InstanceKind::LlvmIntrinsic(_) => { @@ -734,7 +735,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { &Self::copy_fn_args(args), destination, target, - ) + )?; + self.deallocate_moved_locals(caller_moved_locals) } ty::InstanceKind::Shim(ty::ShimKind::VTable(..)) | ty::InstanceKind::Shim(ty::ShimKind::Reify(..)) @@ -764,7 +766,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { unwind, )? else { - return interp_ok(()); + return self.deallocate_moved_locals(caller_moved_locals); }; // Special handling for the closure ABI: untuple the last argument. @@ -799,7 +801,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { &args, with_caller_location, destination, - ReturnContinuation::Goto { ret: target, unwind }, + ReturnContinuation::Goto { ret: target, unwind, caller_moved_locals }, ) } // `InstanceKind::Virtual` does not have callable MIR. Calls to `Virtual` instances must be @@ -887,6 +889,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { FnVal::Instance(fn_inst), (caller_abi, Some(&caller_fn_abi)), &args, + caller_moved_locals, with_caller_location, destination, target, @@ -942,16 +945,19 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // as that "executes" the goto to the return block, but we don't want to, // only the tail called function should return to the current return block. - // The arguments need to all be copied since the current stack frame will be removed - // before the callee even starts executing. - // FIXME(explicit_tail_calls,#144855): does this match what codegen does? - let args = args.iter().map(|fn_arg| FnArg::Copy(fn_arg.copy_fn_arg())).collect::>(); + // Tail-call arguments are evaluated as ordinary operands, so none of them may donate a + // place in the frame that is about to be destroyed. + for arg in args { + debug_assert_matches!(arg, FnArg::Copy(_)); + } // Remove the frame from the stack. let frame = self.pop_stack_frame_raw()?; // Remember where this frame would have returned to. - let ReturnContinuation::Goto { ret, unwind } = frame.return_cont() else { + let ReturnContinuation::Goto { ret, unwind, caller_moved_locals } = frame.return_cont() + else { bug!("can't tailcall as root of the stack"); }; + let (ret, unwind, caller_moved_locals) = (*ret, *unwind, caller_moved_locals.clone()); // There's no return value to deal with! Instead, we forward the old return place // to the new function. // FIXME(explicit_tail_calls): @@ -962,7 +968,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.init_fn_call( fn_val, (caller_abi, caller_fn_abi), - &*args, + args, + caller_moved_locals, with_caller_location, frame.return_place(), ret, @@ -1027,6 +1034,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { FnVal::Instance(instance), (ExternAbi::Rust, Some(fn_abi)), &[FnArg::Copy(arg.into())], + vec![], false, &ret.into(), Some(target), @@ -1080,7 +1088,15 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.copy_op_allow_transmute(&return_op, frame.return_place())?; trace!("return value: {:?}", self.dump_place(frame.return_place())); } - let return_cont = frame.return_cont(); + let return_to = match frame.return_cont() { + ReturnContinuation::Goto { ret, unwind, caller_moved_locals } => { + for &local in caller_moved_locals { + self.deallocate_moved_local(local)?; + } + Some((*ret, *unwind)) + } + ReturnContinuation::Stop { .. } => None, + }; // Finish popping the stack frame. let return_action = self.cleanup_stack_frame(unwinding, frame)?; // Jump to the next block. @@ -1102,20 +1118,20 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Normal return, figure out where to jump. if unwinding { // Follow the unwind edge. - match return_cont { - ReturnContinuation::Goto { unwind, .. } => { + match return_to { + Some((_, unwind)) => { // This must be the very last thing that happens, since it can in fact push a new stack frame. self.unwind_to_block(unwind) } - ReturnContinuation::Stop { .. } => { + None => { panic!("encountered ReturnContinuation::Stop when unwinding!") } } } else { // Follow the normal return edge. - match return_cont { - ReturnContinuation::Goto { ret, .. } => self.return_to_block(ret), - ReturnContinuation::Stop { .. } => { + match return_to { + Some((ret, _)) => self.return_to_block(ret), + None => { assert!( self.stack().is_empty(), "only the bottommost frame can have ReturnContinuation::Stop" diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 8fa028df9455f..e8a0d3a148fd2 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -48,6 +48,11 @@ pub struct InterpCx<'tcx, M: Machine<'tcx>> { /// The virtual memory system. pub memory: Memory<'tcx, M>, + /// Temporary allocations used to preserve values after their source local is moved out. + /// + /// These are cleared after each MIR statement or terminator. + pub(super) move_out_temps: Vec>, + /// The recursion limit (cached from `tcx.recursion_limit(())`) pub recursion_limit: Limit, } @@ -254,6 +259,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { typing_env, layout_cache: RefCell::new(FxHashMap::default()), memory: Memory::new(), + move_out_temps: Vec::new(), recursion_limit: tcx.recursion_limit(), } } diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 0528fee35031c..57e2aa8d7e565 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -165,6 +165,12 @@ pub trait Machine<'tcx>: Sized { /// Whether memory accesses should be alignment-checked. fn enforce_alignment(ecx: &InterpCx<'tcx, Self>) -> bool; + /// Whether to enforce the local allocation semantics required by MIR move elimination. + #[inline(always)] + fn move_elimination_semantics(ecx: &InterpCx<'tcx, Self>) -> bool { + ecx.tcx.sess.opts.unstable_opts.mir_move_elimination + } + /// Gives the machine a chance to detect more misalignment than the built-in checks would catch. #[inline(always)] fn alignment_check( @@ -243,6 +249,7 @@ pub trait Machine<'tcx>: Sized { ecx: &mut InterpCx<'tcx, Self>, instance: ty::Instance<'tcx>, args: &[OpTy<'tcx, Self::Provenance>], + caller_moved_locals: &mut Vec, destination: &PlaceTy<'tcx, Self::Provenance>, target: Option, unwind: mir::UnwindAction, diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 4bafea98cd569..a4d676dba7887 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -829,7 +829,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// by passing it in here. #[inline] pub fn eval_operand( - &self, + &mut self, mir_op: &mir::Operand<'tcx>, layout: Option>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { @@ -838,8 +838,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { use rustc_middle::mir::Operand::*; let op = match mir_op { - // FIXME: do some more logic on `move` to invalidate the old location - &Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?, + &Copy(place) => self.eval_place_to_op(place, layout)?, + &Move(place) if M::move_elimination_semantics(self) && place.projection.is_empty() => { + self.move_out_local(place.local, layout)? + } + &Move(place) => self.eval_place_to_op(place, layout)?, &RuntimeChecks(checks) => { let val = M::runtime_checks(self, checks)?; diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index 4847a0636a5c2..1ee82b7c2ab86 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -643,6 +643,19 @@ where interp_ok(place) } + /// Computes a destination place, allocating its base local if it is currently live but without + /// an allocation. + pub fn eval_place_for_write( + &mut self, + mir_place: mir::Place<'tcx>, + skip_validity_for_simple_deref: bool, + ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { + if M::move_elimination_semantics(self) && !mir_place.is_indirect_first_projection() { + self.allocate_local_for_write(mir_place.local)?; + } + self.eval_place(mir_place, skip_validity_for_simple_deref) + } + /// Given a place, returns either the underlying mplace or a reference to where the value of /// this place is stored. #[inline(always)] diff --git a/compiler/rustc_const_eval/src/interpret/stack.rs b/compiler/rustc_const_eval/src/interpret/stack.rs index d291f1f6fdcbc..ad5be5bf8d3cc 100644 --- a/compiler/rustc_const_eval/src/interpret/stack.rs +++ b/compiler/rustc_const_eval/src/interpret/stack.rs @@ -18,7 +18,7 @@ use tracing::{info_span, instrument, trace}; use super::{ AllocId, CtfeProvenance, FnArg, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, - MemPlaceMeta, MemoryKind, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, + MemPlaceMeta, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, from_known_layout, interp_ok, throw_ub, throw_unsup, }; use crate::{diagnostics, enter_trace_span}; @@ -113,13 +113,19 @@ pub struct Frame<'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> { } /// Where and how to continue when returning/unwinding from the current function. -#[derive(Clone, Copy, Eq, PartialEq, Debug)] // Miri debug-prints these +#[derive(Eq, PartialEq, Debug)] // Miri debug-prints these pub enum ReturnContinuation { /// Jump to the next block in the caller, or cause UB if None (that's a function /// that may never return). /// `ret` stores the block we jump to on a normal return, while `unwind` /// stores the block used for cleanup during unwinding. - Goto { ret: Option, unwind: mir::UnwindAction }, + Goto { + ret: Option, + unwind: mir::UnwindAction, + /// Locals in the caller that were donated to a callee. They are + /// deallocated when the callee returns or unwinds. + caller_moved_locals: Vec, + }, /// The root frame of the stack: nowhere else to jump to, so we stop. /// `cleanup` says whether locals are deallocated. Static computation /// wants them leaked to intern what they need (and just throw away @@ -153,6 +159,8 @@ impl std::fmt::Debug for LocalState<'_, Prov> { pub(super) enum LocalValue { /// This local is not currently alive, and cannot be used at all. Dead, + /// This local is alive, but does not currently have an allocation. + LiveUnallocated, /// A normal, live local. /// Mostly for convenience, we re-use the `Operand` type here. /// This is an optimization over just always having a pointer here; @@ -173,7 +181,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { &self, ) -> Option>, MemPlaceMeta), Immediate>> { match self.value { - LocalValue::Dead => None, + LocalValue::Dead | LocalValue::LiveUnallocated => None, LocalValue::Live(Operand::Indirect(mplace)) => Some(Left((mplace.ptr, mplace.meta))), LocalValue::Live(Operand::Immediate(imm)) => Some(Right(imm)), } @@ -184,6 +192,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { pub(super) fn access(&self) -> InterpResult<'tcx, &Operand> { match &self.value { LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"? + LocalValue::LiveUnallocated => throw_ub!(UnallocatedLocal), LocalValue::Live(val) => interp_ok(val), } } @@ -194,6 +203,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { pub(super) fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand> { match &mut self.value { LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"? + LocalValue::LiveUnallocated => throw_ub!(UnallocatedLocal), LocalValue::Live(val) => interp_ok(val), } } @@ -289,8 +299,8 @@ impl<'tcx, Prov: Provenance, Extra> Frame<'tcx, Prov, Extra> { &self.return_place } - pub fn return_cont(&self) -> ReturnContinuation { - self.return_cont + pub fn return_cont(&self) -> &ReturnContinuation { + &self.return_cont } /// Return the `SourceInfo` of the current instruction. @@ -378,7 +388,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // We can push a `Root` frame if and only if the stack is empty. debug_assert_eq!( self.stack().is_empty(), - matches!(return_cont, ReturnContinuation::Stop { .. }) + matches!(&return_cont, ReturnContinuation::Stop { .. }) ); // First push a stack frame so we have access to `instantiate_from_current_frame` and other @@ -443,14 +453,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { unwinding: bool, frame: Frame<'tcx, M::Provenance, M::FrameExtra>, ) -> InterpResult<'tcx, ReturnAction> { - let return_cont = frame.return_cont; - // Cleanup: deallocate locals. // Usually we want to clean up (deallocate locals), but in a few rare cases we don't. // We do this while the frame is still on the stack, so errors point to the callee. - let cleanup = match return_cont { + let cleanup = match &frame.return_cont { ReturnContinuation::Goto { .. } => true, - ReturnContinuation::Stop { cleanup, .. } => cleanup, + ReturnContinuation::Stop { cleanup, .. } => *cleanup, }; if cleanup { @@ -542,36 +550,49 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } - // This is a hot function, we avoid computing the layout when possible. - // `unsized_` will be `None` for sized types and `Some(layout)` for unsized types. - let unsized_ = if is_very_trivially_sized(self.body().local_decls[local].ty) { - None + // This is a hot function, so normally we avoid computing the layout when possible. Under + // move-elimination semantics we need the layout of every local to identify ZSTs: unlike + // other sized locals, these don't use the LiveUnallocated state.. + let layout = if M::move_elimination_semantics(self) + || !is_very_trivially_sized(self.body().local_decls[local].ty) + { + Some(self.layout_of_local(self.frame(), local, None)?) } else { - // We need the layout. - let layout = self.layout_of_local(self.frame(), local, None)?; - if layout.is_sized() { None } else { Some(layout) } + None }; - - let local_val = LocalValue::Live(if let Some(layout) = unsized_ { - if !meta.has_meta() { - throw_unsup!(UnsizedLocal); - } - // Need to allocate some memory, since `Immediate::Uninit` cannot be unsized. - let dest_place = self.allocate_dyn(layout, MemoryKind::Stack, meta)?; - Operand::Indirect(*dest_place.mplace()) + // `unsized_` will be `None` for sized types and `Some(layout)` for unsized types. + let unsized_ = layout.filter(|layout| layout.is_unsized()); + let is_zst = layout.is_some_and(|layout| layout.is_zst()); + + // `LiveUnallocated` cannot preserve the metadata needed to allocate an unsized local + // later. Unsized locals are only supported as function arguments, where the metadata is + // available here and the local is initialized immediately after being made live, so keep + // allocating them eagerly. + let local_val = if M::move_elimination_semantics(self) && unsized_.is_none() && !is_zst { + assert!(!meta.has_meta()); + LocalValue::LiveUnallocated } else { - // Just make this an efficient immediate. - assert!(!meta.has_meta()); // we're dropping the metadata - // Make sure the machine knows this "write" is happening. (This is important so that - // races involving local variable allocation can be detected by Miri.) - M::after_local_write(self, local, /*storage_live*/ true)?; - // Note that not calling `layout_of` here does have one real consequence: - // if the type is too big, we'll only notice this when the local is actually initialized, - // which is a bit too late -- we should ideally notice this already here, when the memory - // is conceptually allocated. But given how rare that error is and that this is a hot function, - // we accept this downside for now. - Operand::Immediate(Immediate::Uninit) - }); + LocalValue::Live(if let Some(layout) = unsized_ { + if !meta.has_meta() { + throw_unsup!(UnsizedLocal); + } + // Need to allocate some memory, since `Immediate::Uninit` cannot be unsized. + let dest_place = self.allocate_dyn(layout, MemoryKind::Stack, meta)?; + Operand::Indirect(*dest_place.mplace()) + } else { + // Just make this an efficient immediate. + assert!(!meta.has_meta()); // we're dropping the metadata + // Make sure the machine knows this "write" is happening. (This is important so that + // races involving local variable allocation can be detected by Miri.) + M::after_local_write(self, local, /*storage_live*/ true)?; + // Note that not calling `layout_of` here does have one real consequence: + // if the type is too big, we'll only notice this when the local is actually initialized, + // which is a bit too late -- we should ideally notice this already here, when the memory + // is conceptually allocated. But given how rare that error is and that this is a hot function, + // we accept this downside for now. + Operand::Immediate(Immediate::Uninit) + }) + }; // If the local is already live, deallocate its old memory. let old = mem::replace(&mut self.frame_mut().locals[local].value, local_val); @@ -595,6 +616,76 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { interp_ok(()) } + /// Ensure that a direct destination local has an allocation. + pub(super) fn allocate_local_for_write(&mut self, local: mir::Local) -> InterpResult<'tcx> { + let local_value = &mut self.frame_mut().locals[local].value; + if matches!(local_value, LocalValue::LiveUnallocated) { + *local_value = LocalValue::Live(Operand::Immediate(Immediate::Uninit)); + M::after_local_write(self, local, /*storage_live*/ true)?; + } + interp_ok(()) + } + + /// Move an entire local into a detached value and leave the local live but unallocated. + pub(super) fn move_out_local( + &mut self, + local: mir::Local, + layout: Option>, + ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + let op = self.local_to_op(local, layout)?; + // ZST storage remains allocated until `StorageDead`. + if op.layout.is_zst() { + return interp_ok(op); + } + let moved_op = match *op.op() { + Operand::Immediate(_) => op, + Operand::Indirect(_) => { + // We need to free the local's allocation immediately to catch + // any use-after-move, so move the contents to a temporary + // allocation for the duration of the statement. + let temp = self.allocate(op.layout, MemoryKind::Stack)?; + self.copy_op_no_validate(&op, &temp, /*allow_transmute*/ false)?; + self.move_out_temps.push(temp.clone()); + temp.into() + } + }; + + let old = + mem::replace(&mut self.frame_mut().locals[local].value, LocalValue::LiveUnallocated); + self.deallocate_local(old)?; + interp_ok(moved_op) + } + + /// Deallocate a local that was donated to a callee for the duration of a call. + pub(super) fn deallocate_moved_local(&mut self, local: mir::Local) -> InterpResult<'tcx> { + let old = + mem::replace(&mut self.frame_mut().locals[local].value, LocalValue::LiveUnallocated); + match old { + LocalValue::Live(_) => self.deallocate_local(old), + LocalValue::Dead | LocalValue::LiveUnallocated => { + bug!("call argument local was not allocated") + } + } + } + + pub(super) fn deallocate_moved_locals( + &mut self, + locals: Vec, + ) -> InterpResult<'tcx> { + for local in locals { + self.deallocate_moved_local(local)?; + } + interp_ok(()) + } + + /// Deallocate temporary allocations created by whole-local moves in the current MIR step. + pub(super) fn clear_move_out_temps(&mut self) -> InterpResult<'tcx> { + for temp in mem::take(&mut self.move_out_temps) { + self.deallocate_ptr(temp.ptr(), None, MemoryKind::Stack)?; + } + interp_ok(()) + } + fn deallocate_local(&mut self, local: LocalValue) -> InterpResult<'tcx> { if let LocalValue::Live(Operand::Indirect(MemPlace { ptr, .. })) = local { // All locals have a backing allocation, even if the allocation is empty @@ -697,6 +788,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> { ) -> std::fmt::Result { match self.value { LocalValue::Dead => write!(fmt, " is dead")?, + LocalValue::LiveUnallocated => write!(fmt, " is live but unallocated")?, LocalValue::Live(Operand::Immediate(Immediate::Uninit)) => { write!(fmt, " is uninitialized")? } diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 6dd1ed598e3aa..c338cea269c1b 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -24,6 +24,7 @@ use crate::{enter_trace_span, util}; struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> { callee: FnVal<'tcx, M::ExtraFnVal>, args: Vec>, + caller_moved_locals: Vec, fn_sig: ty::FnSig<'tcx>, /// None if LLVM intrinsic fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>, @@ -55,6 +56,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { if let Some(stmt) = basic_block.statements.get(loc.statement_index) { let old_frames = self.frame_idx(); self.eval_statement(stmt)?; + self.clear_move_out_temps()?; // Make sure we are not updating `statement_index` of the wrong frame. assert_eq!(old_frames, self.frame_idx()); // Advance the program counter. @@ -66,6 +68,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let terminator = basic_block.terminator(); self.eval_terminator(terminator)?; + self.clear_move_out_temps()?; if !self.stack().is_empty() { if let Either::Left(loc) = self.frame().loc { info!("// executing {:?}", loc.block); @@ -94,8 +97,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Assign((place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?, SetDiscriminant { place, variant_index } => { - let dest = - self.eval_place(**place, /* skip_validity_for_simple_deref */ false)?; + let dest = self.eval_place_for_write( + **place, /* skip_validity_for_simple_deref */ false, + )?; self.write_discriminant(*variant_index, &dest)?; } @@ -162,10 +166,47 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { rvalue: &mir::Rvalue<'tcx>, place: mir::Place<'tcx>, ) -> InterpResult<'tcx> { - // We can skip validity because we'll write to the place which checks everything we care - // about for references, and the pointee must be sized so there's nothing to check for raw - // pointers. - let dest = self.eval_place(place, /* skip_validity_for_simple_deref */ true)?; + if M::move_elimination_semantics(self) { + // Create a temporary allocation to hold the result of evaluating the rvalue. + let ty = self.instantiate_from_current_frame_and_normalize_erasing_regions( + place.ty(&self.frame().body.local_decls, *self.tcx).ty, + )?; + let layout = self.layout_of(ty)?; + let temp = self.allocate(layout, super::MemoryKind::Stack)?; + + // Evaluate the rvalue into the temporary allocation. + self.eval_rvalue_into_resolved_place(rvalue, temp.clone().into())?; + + // Evaluate the destination place after all source operands have been evaluated. This is + // important since it allows the destination to reuse the address of a moved operand. It + // also ensures that the destination local isn't incorrectly deallocated as part of move + // operand evaluation. + // + // See the comment below for skip_validity_for_simple_deref. + let dest = + self.eval_place_for_write(place, /* skip_validity_for_simple_deref */ true)?; + + // Copy the evaluated rvalue to the destination place and release the temporary + // allocation. + self.copy_op_no_validate(&temp, &dest, /* allow_transmute */ false)?; + self.deallocate_ptr(temp.ptr(), None, super::MemoryKind::Stack)?; + } else { + // We can skip validity because we'll write to the place which checks everything we care + // about for references, and the pointee must be sized so there's nothing to check for raw + // pointers. + let dest = + self.eval_place_for_write(place, /* skip_validity_for_simple_deref */ true)?; + self.eval_rvalue_into_resolved_place(rvalue, dest)?; + } + + interp_ok(()) + } + + fn eval_rvalue_into_resolved_place( + &mut self, + rvalue: &mir::Rvalue<'tcx>, + dest: PlaceTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx> { // FIXME: ensure some kind of non-aliasing between LHS and RHS? // Also see https://github.com/rust-lang/rust/issues/68364. @@ -187,9 +228,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { BinaryOp(bin_op, (ref left, ref right)) => { let layout = util::binop_left_homogeneous(bin_op).then_some(dest.layout); - let left = self.read_immediate(&self.eval_operand(left, layout)?)?; + let left = self.eval_operand(left, layout)?; + let left = self.read_immediate(&left)?; let layout = util::binop_right_homogeneous(bin_op).then_some(left.layout); - let right = self.read_immediate(&self.eval_operand(right, layout)?)?; + let right = self.eval_operand(right, layout)?; + let right = self.read_immediate(&right)?; let result = self.binary_op(bin_op, &left, &right)?; assert_eq!(result.layout, dest.layout, "layout mismatch for result of {bin_op:?}"); self.write_immediate(*result, &dest)?; @@ -197,7 +240,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { UnaryOp(un_op, ref operand) => { let layout = util::unop_homogeneous(un_op).then_some(dest.layout); - let val = self.read_immediate(&self.eval_operand(operand, layout)?)?; + let val = self.eval_operand(operand, layout)?; + let val = self.read_immediate(&val)?; let result = self.unary_op(un_op, &val)?; assert_eq!(result.layout, dest.layout, "layout mismatch for result of {un_op:?}"); self.write_immediate(*result, &dest)?; @@ -399,11 +443,15 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { interp_ok(()) } - /// Evaluate the arguments of a function call + /// Evaluate the arguments of a function call. + /// + /// This is not used for tail calls: those always use normal operand + /// evaluation since they cannot use `FnArg::InPlace`. fn eval_fn_call_argument( &mut self, op: &mir::Operand<'tcx>, move_definitely_disjoint: bool, + caller_moved_locals: &mut Vec, ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> { interp_ok(match op { mir::Operand::Copy(_) | mir::Operand::Constant(_) | mir::Operand::RuntimeChecks(_) => { @@ -411,11 +459,18 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let op = self.eval_operand(op, None)?; FnArg::Copy(op) } - mir::Operand::Move(place) => { + mir::Operand::Move(mir_place) => { // We will read from this place, which checks everything there is to check, // so we can skip the extra validity check here. let place = - self.eval_place(*place, /* skip_validity_for_simple_deref */ true)?; + self.eval_place(*mir_place, /* skip_validity_for_simple_deref */ true)?; + // Fully moved non-ZST locals are deallocated upon return.. + if M::move_elimination_semantics(self) + && mir_place.projection.is_empty() + && !place.layout.is_zst() + { + caller_moved_locals.push(mir_place.local); + } if move_definitely_disjoint { // We still have to ensure that no *other* pointers are used to access this place, // so *if* it is in memory then we have to treat it as `InPlace`. @@ -438,46 +493,61 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn eval_callee_and_args( &mut self, terminator: &mir::Terminator<'tcx>, + is_tail_call: bool, func: &mir::Operand<'tcx>, args: &[Spanned>], dest: &mir::Place<'tcx>, ) -> InterpResult<'tcx, EvaluatedCalleeAndArgs<'tcx, M>> { let func = self.eval_operand(func, None)?; - // Evaluating function call arguments. The tricky part here is dealing with `Move` - // arguments: we have to ensure no two such arguments alias. This would be most easily done - // by just forcing them all into memory and then doing the usual in-place argument - // protection, but then we'd force *a lot* of arguments into memory. So we do some syntactic - // pre-processing here where if all `move` arguments are syntactically distinct local - // variables (and none is indirect), we can skip the in-memory forcing. - // We have to include `dest` in that list so that we can detect aliasing of an in-place - // argument with the return place. - let move_definitely_disjoint = 'move_definitely_disjoint: { - let mut previous_locals = FxHashSet::::default(); - for place in args - .iter() - .filter_map(|a| { - // We only have to care about `Move` arguments. - if let mir::Operand::Move(place) = &a.node { Some(place) } else { None } - }) - .chain(iter::once(dest)) - { - if place.is_indirect_first_projection() { - // An indirect in-place argument could alias with anything else... - break 'move_definitely_disjoint false; - } - if !previous_locals.insert(place.local) { - // This local is the base for two arguments! They might overlap. - break 'move_definitely_disjoint false; + let mut caller_moved_locals = vec![]; + let args = if is_tail_call { + // The current frame is destroyed by a tail call, so its argument places cannot be + // donated to the callee. Evaluate them as ordinary operands instead. + args.iter() + .map(|arg| self.eval_operand(&arg.node, None).map(FnArg::Copy)) + .collect::>>()? + } else { + // Evaluating function call arguments. The tricky part here is dealing with `Move` + // arguments: we have to ensure no two such arguments alias. This would be most easily + // done by just forcing them all into memory and then doing the usual in-place argument + // protection, but then we'd force *a lot* of arguments into memory. So we do some + // syntactic pre-processing here where if all `move` arguments are syntactically + // distinct local variables (and none is indirect), we can skip the in-memory forcing. + // We have to include `dest` in that list so that we can detect aliasing of an in-place + // argument with the return place. + let move_definitely_disjoint = 'move_definitely_disjoint: { + let mut previous_locals = FxHashSet::::default(); + for place in args + .iter() + .filter_map(|a| { + // We only have to care about `Move` arguments. + if let mir::Operand::Move(place) = &a.node { Some(place) } else { None } + }) + .chain(iter::once(dest)) + { + if place.is_indirect_first_projection() { + // An indirect in-place argument could alias with anything else... + break 'move_definitely_disjoint false; + } + if !previous_locals.insert(place.local) { + // This local is the base for two arguments! They might overlap. + break 'move_definitely_disjoint false; + } } - } - // We found no violation so they are all definitely disjoint. - true + // We found no violation so they are all definitely disjoint. + true + }; + args.iter() + .map(|arg| { + self.eval_fn_call_argument( + &arg.node, + move_definitely_disjoint, + &mut caller_moved_locals, + ) + }) + .collect::>>()? }; - let args = args - .iter() - .map(|arg| self.eval_fn_call_argument(&arg.node, move_definitely_disjoint)) - .collect::>>()?; let fn_sig_binder = { let _trace = enter_trace_span!(M, "fn_sig", ty = ?func.layout.ty.kind()); @@ -513,7 +583,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } }; - interp_ok(EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location }) + interp_ok(EvaluatedCalleeAndArgs { + callee, + args, + caller_moved_locals, + fn_sig, + fn_abi, + with_caller_location, + }) } fn eval_terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> { @@ -535,7 +612,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Goto { target } => self.go_to_block(target), SwitchInt { ref discr, ref targets } => { - let discr = self.read_immediate(&self.eval_operand(discr, None)?)?; + let discr = self.eval_operand(discr, None)?; + let discr = self.read_immediate(&discr)?; trace!("SwitchInt({:?})", *discr); // Branch to the `otherwise` case by default, if no match is found. @@ -570,16 +648,49 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let old_stack = self.frame_idx(); let old_loc = self.frame().loc; - // Evaluation order consistent with assignment: destination first. - let dest_place = - self.eval_place(destination, /* skip_validity_for_simple_deref */ false)?; - let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = - self.eval_callee_and_args(terminator, func, args, &destination)?; + let (dest_place, evaluated) = if M::move_elimination_semantics(self) { + // With move-elimination semantics, evaluate the destination last. + let evaluated = self.eval_callee_and_args( + terminator, + /* is_tail_call */ false, + func, + args, + &destination, + )?; + let dest_place = self.eval_place_for_write( + destination, + /* skip_validity_for_simple_deref */ false, + )?; + (dest_place, evaluated) + } else { + // Without move-elimination semantics, evaluate the destination first. + let dest_place = self.eval_place_for_write( + destination, + /* skip_validity_for_simple_deref */ false, + )?; + let evaluated = self.eval_callee_and_args( + terminator, + /* is_tail_call */ false, + func, + args, + &destination, + )?; + (dest_place, evaluated) + }; + let EvaluatedCalleeAndArgs { + callee, + args, + caller_moved_locals, + fn_sig, + fn_abi, + with_caller_location, + } = evaluated; self.init_fn_call( callee, (fn_sig.abi(), fn_abi), &args, + caller_moved_locals, with_caller_location, &dest_place, target, @@ -603,8 +714,21 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { TailCall { ref func, ref args, fn_span: _ } => { let old_frame_idx = self.frame_idx(); - let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = - self.eval_callee_and_args(terminator, func, args, &mir::Place::return_place())?; + let EvaluatedCalleeAndArgs { + callee, + args, + caller_moved_locals, + fn_sig, + fn_abi, + with_caller_location, + } = self.eval_callee_and_args( + terminator, + /* is_tail_call */ true, + func, + args, + &mir::Place::return_place(), + )?; + debug_assert!(caller_moved_locals.is_empty()); self.init_fn_tail_call( callee, @@ -648,7 +772,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Assert { ref cond, expected, ref msg, target, unwind } => { let ignored = M::ignore_optional_overflow_checks(self) && msg.is_optional_overflow_check(); - let cond_val = self.read_scalar(&self.eval_operand(cond, None)?)?.to_bool()?; + let cond = self.eval_operand(cond, None)?; + let cond_val = self.read_scalar(&cond)?.to_bool()?; if ignored || expected == cond_val { self.go_to_block(target); } else { diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index b7b1531e50857..475c81e66ab09 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -1,6 +1,7 @@ use std::iter::Step; use std::marker::PhantomData; -use std::ops::{Bound, Range, RangeBounds}; +use std::ops::{Bound, RangeBounds}; +use std::range::RangeInclusive; use smallvec::SmallVec; @@ -59,11 +60,14 @@ impl IntervalSet { } /// Iterates through intervals stored in the set, in order. - pub fn iter_intervals(&self) -> impl Iterator> + pub fn iter_intervals(&self) -> impl Iterator> where I: Step, { - self.map.iter().map(|&(start, end)| I::new(start as usize)..I::new(end as usize + 1)) + self.map.iter().map(|&(start, end)| RangeInclusive { + start: I::new(start as usize), + last: I::new(end as usize), + }) } /// Returns true if we increased the number of elements present. @@ -204,17 +208,38 @@ impl IntervalSet { needle <= *prev_end } + /// Returns whether any point in `range` is contained in the set. + pub fn intersects_range(&self, range: impl RangeBounds + Clone) -> bool { + let start = inclusive_start(range.clone()); + let Some(end) = inclusive_end(self.domain, range) else { + // empty range + return false; + }; + if start > end { + return false; + } + + // Find the last interval whose start is <= end. + let Some(last) = self.map.partition_point(|r| r.0 <= end).checked_sub(1) else { + // All ranges in the map start after the new range's end + return false; + }; + let (_, prev_end) = &self.map[last]; + start <= *prev_end + } + pub fn superset(&self, other: &IntervalSet) -> bool where I: Step, { let mut sup_iter = self.iter_intervals(); let mut current = None; - let contains = |sup: Range, sub: Range, current: &mut Option>| { - if sup.end < sub.start { - // if `sup.end == sub.start`, the next sup doesn't contain `sub.start` + let contains = |sup: RangeInclusive, + sub: RangeInclusive, + current: &mut Option>| { + if sup.last < sub.start { None // continue to the next sup - } else if sup.end >= sub.end && sup.start <= sub.start { + } else if sup.last >= sub.last && sup.start <= sub.start { *current = Some(sup); // save the current sup Some(true) } else { @@ -224,8 +249,8 @@ impl IntervalSet { other.iter_intervals().all(|sub| { current .take() - .and_then(|sup| contains(sup, sub.clone(), &mut current)) - .or_else(|| sup_iter.find_map(|sup| contains(sup, sub.clone(), &mut current))) + .and_then(|sup| contains(sup, sub, &mut current)) + .or_else(|| sup_iter.find_map(|sup| contains(sup, sub, &mut current))) .unwrap_or(false) }) } @@ -242,11 +267,11 @@ impl IntervalSet { let mut other_current = other_iter.next()?; loop { - if self_current.end <= other_current.start { + if self_current.last < other_current.start { self_current = self_iter.next()?; continue; } - if other_current.end <= self_current.start { + if other_current.last < self_current.start { other_current = other_iter.next()?; continue; } @@ -374,6 +399,12 @@ impl SparseIntervalMatrix { self.rows.iter_enumerated() } + pub fn clear_row(&mut self, row: R) { + if let Some(row) = self.rows.get_mut(row) { + row.clear(); + } + } + fn ensure_row(&mut self, row: R) -> &mut IntervalSet { self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size)) } @@ -397,6 +428,16 @@ impl SparseIntervalMatrix { write_row.union(read_row) } + pub fn disjoint_rows(&self, a: R, b: R) -> bool + where + C: Step, + { + let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else { + return true; + }; + a.disjoint(b) + } + pub fn insert_all_into_row(&mut self, row: R) { self.ensure_row(row).insert_all(); } diff --git a/compiler/rustc_index/src/interval/tests.rs b/compiler/rustc_index/src/interval/tests.rs index 375af60f66207..cf3222e6c6572 100644 --- a/compiler/rustc_index/src/interval/tests.rs +++ b/compiler/rustc_index/src/interval/tests.rs @@ -5,7 +5,7 @@ fn insert_collapses() { let mut set = IntervalSet::::new(10000); set.insert_range(9831..=9837); set.insert_range(43..=9830); - assert_eq!(set.iter_intervals().collect::>(), [43..9838]); + assert_eq!(set.iter_intervals().collect::>(), [(43..=9837).into()]); } #[test] diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 15d7a1609c67f..4fb223f9ab4a6 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -849,6 +849,7 @@ fn test_unstable_options_tracking_hash() { tracked!(min_function_alignment, Some(Align::EIGHT)); tracked!(min_recursion_limit, Some(256)); tracked!(mir_enable_passes, vec![("DestProp".to_string(), false)]); + tracked!(mir_move_elimination, true); tracked!(mir_opt_level, Some(4)); tracked!(mir_preserve_ub, true); tracked!(move_size_limit, Some(4096)); diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs index fb82f694d6f74..40d7331e57345 100644 --- a/compiler/rustc_middle/src/mir/interpret/error.rs +++ b/compiler/rustc_middle/src/mir/interpret/error.rs @@ -410,6 +410,8 @@ pub enum UndefinedBehaviorInfo<'tcx> { InvalidUninitBytes(Option<(AllocId, BadBytesAccess)>), /// Working with a local that is not currently live. DeadLocal, + /// Working with a local that is live but does not currently have an allocation. + UnallocatedLocal, /// A discriminant of an uninhabited enum variant is written. UninhabitedEnumVariantWritten(VariantIdx), /// An uninhabited enum variant is projected. @@ -616,6 +618,7 @@ impl<'tcx> fmt::Display for UndefinedBehaviorInfo<'tcx> { uninit = info.bad, ), DeadLocal => write!(f, "accessing a dead local variable"), + UnallocatedLocal => write!(f, "accessing a live but unallocated local variable"), UninhabitedEnumVariantWritten(_) => { write!(f, "writing discriminant of an uninhabited enum variant") } diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 4e2d16625266c..8284e21ea7bd9 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -124,7 +124,6 @@ pub enum RuntimePhase { /// disallowed: /// * [`TerminatorKind::Yield`] /// * [`TerminatorKind::CoroutineDrop`] - /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array` /// * [`Rvalue::CopyForDeref`] /// * [`PlaceElem::OpaqueCast`] /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp) @@ -783,7 +782,11 @@ pub enum TerminatorKind<'tcx> { /// The evaluation order is currently "first compute destination place, then `func` operand, /// then the arguments in left-to-right order". /// + /// RFC 3943 semantics (enabled with -Z mir-move-elimination) changes the + /// evaluation order to evaluate the destination place last instead. + /// /// [#71117]: https://github.com/rust-lang/rust/issues/71117 + /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943 Call { /// The function that’s being called. func: Operand<'tcx>, @@ -1426,9 +1429,6 @@ pub enum Rvalue<'tcx> { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(Box>, IndexVec>), /// A CopyForDeref is equivalent to a read from a place at the diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs index f8eeb9dfcb43c..cb847cf8f10f9 100644 --- a/compiler/rustc_mir_dataflow/src/framework/direction.rs +++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs @@ -127,6 +127,8 @@ impl Direction for Backward { analysis.apply_primary_statement_effect(state, stmt, loc); vis.visit_after_primary_statement_effect(state, stmt, loc); } + + vis.visit_block_exit(state, block); } } @@ -242,5 +244,7 @@ impl Direction for Forward { vis.visit_after_early_terminator_effect(state, term, loc); analysis.apply_primary_terminator_effect(state, term, loc); vis.visit_after_primary_terminator_effect(state, term, loc); + + vis.visit_block_exit(state, block); } } diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs index e4b840a73e502..5b8a3374e04d9 100644 --- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs +++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs @@ -34,6 +34,13 @@ pub trait ResultsVisitor<'tcx, A> where A: Analysis<'tcx>, { + /// Called after all effects in a block have been applied in the direction + /// of the analysis. + /// + /// In a forwards analysis, `state` is from the block's end. In a backwards + /// analysis, `state` is from the block's start. + fn visit_block_exit(&mut self, _state: &A::Domain, _block: BasicBlock) {} + /// Called after the "early" effect of the given statement is applied to `state`. fn visit_after_early_statement_effect( &mut self, diff --git a/compiler/rustc_mir_dataflow/src/impls/mod.rs b/compiler/rustc_mir_dataflow/src/impls/mod.rs index 1e12e41ce1fb4..495858c776b09 100644 --- a/compiler/rustc_mir_dataflow/src/impls/mod.rs +++ b/compiler/rustc_mir_dataflow/src/impls/mod.rs @@ -1,6 +1,7 @@ mod borrowed_locals; mod initialized; mod liveness; +mod precise_liveness; mod storage_liveness; pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals}; @@ -11,6 +12,9 @@ pub use self::initialized::{ pub use self::liveness::{ DefUse, LivenessTransferFunction, MaybeLiveLocals, MaybeTransitiveLiveLocals, }; +pub use self::precise_liveness::{ + SplitPointEffect, SplitPointIndex, dump_liveness_matrix, liveness_matrix, +}; pub use self::storage_liveness::{ MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals, }; diff --git a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs new file mode 100644 index 0000000000000..af3dc12adae79 --- /dev/null +++ b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs @@ -0,0 +1,571 @@ +//! Computes the points where each local must have a distinct allocation. +//! +//! The result is a [`SparseIntervalMatrix`] with one row per local. Two locals +//! may share the same address only if their rows are disjoint. To model MIR +//! statements where a source operand and destination place may share an +//! address, each statement and terminator is split into an early point, where +//! operands are read, and a late point, where destinations are written. +//! +//! A local live range starts at the late point of any statement or terminator +//! that writes to it without a `Deref` projection. It ends at the early point +//! of a `StorageDead`, a whole-local move operand, or the last use of that +//! local on a control-flow path (only for locals whose address is never +//! observed). +//! +//! `Call` terminators are handled specially: move operands are kept live +//! through the late point of the terminator so they conflict with each other +//! and with the destination place. This matches the runtime behavior where the +//! place is donated to the callee for the duration of the call. + +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{ + MutatingUseContext, NonMutatingUseContext, PlaceContext, VisitPlacesWith, Visitor, +}; +use rustc_middle::mir::{self, BasicBlock, Local, Location, MirDumper, PassWhere, Place}; +use rustc_middle::ty::TyCtxt; +use tracing::trace; + +use crate::impls::{DefUse, MaybeLiveLocals, borrowed_locals}; +use crate::points::{DenseLocationMap, PointIndex}; +use crate::{Analysis, GenKill, ResultsVisitor, visit_results}; + +//////////////////////////////////////////////////////////////////////////////// +// Backward dataflow pass +// +// This pass computes "kill points" for each local, indicating the location of +// their last use in a particular control flow branch. These are later used in +// the forward pass later to end the live range of locals that are never +// borrowed at their last direct use. +// +// Borrowed locals are treated as always live by this pass since those need to +// remain allocated until `StorageDead` or a whole-local move. +// +// This pass has 2 outputs: a set of kill points that mark the last use +// locations of locals and a per-block bitset indicating which locals are live +// on entry to that block. + +struct KillPoints<'a> { + live_on_entry: IndexVec>, + kill_points_map: IndexVec, +} + +impl<'a> KillPoints<'a> { + fn compute<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: Option<&'static str>, + points: &DenseLocationMap, + kill_points: &'a mut Vec<(Local, Location)>, + ) -> Self { + let maybe_live_locals = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, pass_name); + let borrowed_locals = borrowed_locals(body); + + // Initialize all borrowed locals as live on entry. We never try to kill + // those. + let mut live_on_entry = + IndexVec::from_elem_n(borrowed_locals.clone(), body.basic_blocks.len()); + + // Collect kill points and live-on-entry states from the results of + // MaybeLiveLocals. + kill_points.clear(); + let mut visitor = KillPointsVisitor { + kill_points, + live_on_entry: &mut live_on_entry, + borrowed_locals: &borrowed_locals, + }; + visit_results( + body, + mir::traversal::reachable(body).map(|(block, _)| block), + &maybe_live_locals, + &mut visitor, + ); + trace!(?kill_points); + trace!(?live_on_entry); + + // Create a mapping of `PointIndex` to the set of killed locals at that + // location. + let mut kill_points_map = IndexVec::from_elem_n(&[][..], points.num_points()); + for chunk in kill_points.chunk_by(|a, b| a.1 == b.1) { + let point = points.point_from_location(chunk[0].1); + trace!("Kill points at {:?}: {:?}", chunk[0].1, chunk); + kill_points_map[point] = chunk; + } + + Self { live_on_entry, kill_points_map } + } +} + +struct KillPointsVisitor<'a> { + kill_points: &'a mut Vec<(Local, Location)>, + live_on_entry: &'a mut IndexVec>, + borrowed_locals: &'a DenseBitSet, +} + +impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { + fn visit_block_exit(&mut self, state: &DenseBitSet, block: BasicBlock) { + // Borrowed locals are already marked as live when live_on_entry was + // initialized. This adds the non-borrowed locals that we have + // determined are live on entry to this block. + self.live_on_entry[block].union(state); + } + + fn visit_after_early_statement_effect( + &mut self, + state: &DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses. + match ctxt { + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + PlaceContext::NonUse(_) => return, + } + + // If a local is used in a statement but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_statement(statement, location); + } + + fn visit_after_early_terminator_effect( + &mut self, + state: &DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses (they don't do anything) and edge uses + // (implicitly killed though live_on_entry at the start of the + // corresponding successor). + match ctxt { + PlaceContext::MutatingUse( + MutatingUseContext::AsmOutput + | MutatingUseContext::Call + | MutatingUseContext::Yield, + ) + | PlaceContext::NonUse(_) => return, + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + } + + // If a local is used in a terminator but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_terminator(terminator, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Forward dataflow pass + +struct PreciseLiveness<'a> { + kill_points: &'a KillPoints<'a>, + points: &'a DenseLocationMap, +} + +impl PreciseLiveness<'_> { + fn apply_block_start_effect(&self, state: &mut DenseBitSet, block: BasicBlock) { + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.intersect(&self.kill_points.live_on_entry[block]); + } +} + +impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { + type Domain = DenseBitSet; + + const NAME: &'static str = "precise_liveness"; + + fn bottom_value(&self, body: &mir::Body<'tcx>) -> DenseBitSet { + DenseBitSet::new_empty(body.local_decls.len()) + } + + fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut DenseBitSet) { + // Function arguments start out as live. + for arg in body.args_iter() { + state.gen_(arg); + } + } + + fn apply_primary_statement_effect( + &self, + state: &mut DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + // StorageDead always kills a local, even if it has been borrowed. + if let mir::StatementKind::StorageDead(local) = statement.kind { + state.kill(local); + return; + } + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { + if let Some(local) = place.as_local() { + state.kill(local); + } + } + }) + .visit_statement(statement, location); + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => state.gen_(place.local), + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + + // Apply kill points at this statement: if a variable is dead then it + // doesn't need storage. + let point = self.points.point_from_location(location); + for &(local, _) in self.kill_points.kill_points_map[point] { + state.kill(local); + } + } + + fn apply_primary_terminator_effect( + &self, + state: &mut DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) = ctxt { + if let Some(local) = place.as_local() { + state.kill(local); + } + } + }) + .visit_terminator(terminator, location); + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // These are handled through `apply_call_return_effect`. + if let PlaceContext::MutatingUse( + MutatingUseContext::AsmOutput + | MutatingUseContext::Call + | MutatingUseContext::Yield, + ) = ctxt + { + return; + } + + match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => state.gen_(place.local), + DefUse::Use | DefUse::NonUse => {} + } + }) + .visit_terminator(terminator, location); + } + + fn apply_call_return_effect( + &self, + state: &mut DenseBitSet, + _block: BasicBlock, + return_places: mir::CallReturnPlaces<'_, 'tcx>, + ) { + return_places.for_each(|place| state.gen_(place.local)); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Matrix construction + +/// Different "phases" of a single MIR statement, used to describe how +/// overlapping operands are handled. +/// +/// As a general rule, source operands are read in the `Early` phase and +/// destination places are written in the `Late` phase. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum SplitPointEffect { + Early = 0, + Late = 1, +} + +rustc_index::newtype_index! { + /// A `PointIndex` with the lower bit encoding early/late inside a + /// statement. + /// + /// This is used to model overlap constraints within a MIR statement: if a + /// source/destination are allowed to overlap then the source is read in + /// `SplitPointEffect::Early` and the write is done in + /// `SplitPointEffect::Late`. + #[orderable] + #[debug_format = "SplitPointIndex({})"] + pub struct SplitPointIndex {} +} + +impl SplitPointIndex { + pub fn new(point: PointIndex, effect: SplitPointEffect) -> SplitPointIndex { + let index = (point.as_u32() << 1) | (effect as u32); + SplitPointIndex::from_u32(index) + } + + pub fn point(self) -> PointIndex { + PointIndex::from_u32(self.as_u32() >> 1) + } + + pub fn effect(self) -> SplitPointEffect { + match self.as_u32() & 1 { + 0 => SplitPointEffect::Early, + 1 => SplitPointEffect::Late, + _ => unreachable!(), + } + } +} + +/// Helper type to construct a `SparseIntervalMatrix`. +struct MatrixBuilder { + matrix: SparseIntervalMatrix, + range_start: IndexVec>, + + // Track locals that have been live at any point in a block so that at the + // end of a block we don't need to iterate over all locals. This + // significantly speeds up matrix building. + maybe_live_locals: Vec, +} + +impl MatrixBuilder { + fn gen_(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let split_point = SplitPointIndex::new(point, effect); + + // No-op if the local is already live. + if self.range_start[local].is_none() { + self.range_start[local] = Some(split_point); + self.maybe_live_locals.push(local); + } + } + + fn kill(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let end = SplitPointIndex::new(point, effect); + + // No-op if the local is already dead. + if let Some(start) = self.range_start[local].take() { + debug_assert!(end >= start); + self.matrix.append_range(local, start..=end); + } + } + + fn kill_all(&mut self, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + self.kill(local, point, effect); + } + } + + fn kill_all_except(&mut self, except: Local, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + if local != except { + self.kill(local, point, effect); + } + } + self.maybe_live_locals.push(except); + } +} + +pub fn liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + points: &DenseLocationMap, + pass_name: Option<&'static str>, +) -> SparseIntervalMatrix { + let mut kill_points_vec = vec![]; + let kill_points = KillPoints::compute(tcx, body, pass_name, points, &mut kill_points_vec); + let mut results = PreciseLiveness { kill_points: &kill_points, points } + .iterate_to_fixpoint(tcx, body, pass_name); + + let mut builder = MatrixBuilder { + matrix: SparseIntervalMatrix::new(points.num_points() * 2), + range_start: IndexVec::from_elem_n(None, body.local_decls.len()), + maybe_live_locals: Vec::new(), + }; + for (block, block_data) in body.basic_blocks.iter_enumerated() { + // We can mutate the state in-place since we're not using it any more + // after this point. + let state = &mut results.entry_states[block]; + + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.intersect(&kill_points.live_on_entry[block]); + + // Gen any locals that are live at the start of the block. If this block + // only consists of a return terminator then instead of gen the return + // place. This ensures that StorageDead for all other locals are + // inserted before the return terminator. + let terminator = block_data.terminator(); + if let mir::TerminatorKind::Return = terminator.kind + && block_data.statements.is_empty() + { + if state.contains(mir::RETURN_PLACE) { + builder.gen_(mir::RETURN_PLACE, points.entry_point(block), SplitPointEffect::Early); + } + } else { + for local in state.iter() { + builder.gen_(local, points.entry_point(block), SplitPointEffect::Early); + } + } + + for (statement_index, statement) in block_data.statements.iter().enumerate() { + let location = Location { block, statement_index }; + let point = points.point_from_location(location); + + // StorageDead always kills a local, even if it has been borrowed. + if let mir::StatementKind::StorageDead(local) = statement.kind { + builder.kill(local, point, SplitPointEffect::Late); + continue; + } + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { + if let Some(local) = place.as_local() { + builder.kill(local, point, SplitPointEffect::Early); + } + } + }) + .visit_statement(statement, location); + + // Kill any locals which are no longer used after this statement. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + builder.gen_(place.local, point, SplitPointEffect::Late) + } + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + + // Kill any dead destination places: they will only appear at the + // late point of the statement they are generated in, which is + // sufficient for determining overlap. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Late); + } + } + + // If this block ends in a return terminator, end all live ranges before + // the terminator so that StorageDead statements are inserted before it. + // + // This is useful after inlining so that the lifetime of locals in the + // inlined callee don't extend past the call in the callee. + if let mir::TerminatorKind::Return = terminator.kind + && !block_data.statements.is_empty() + { + // Blocks with only a return terminator are handled above. + let location = Location { block, statement_index: block_data.statements.len() - 1 }; + let point = points.point_from_location(location); + builder.kill_all_except(mir::RETURN_PLACE, point, SplitPointEffect::Late); + } + + let location = Location { block, statement_index: block_data.statements.len() }; + let point = points.point_from_location(location); + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) = ctxt { + if let Some(local) = place.as_local() { + builder.kill(local, point, SplitPointEffect::Early); + } + } + }) + .visit_terminator(terminator, location); + + // Kill any locals which are no longer used after this terminator. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + builder.gen_(place.local, point, SplitPointEffect::Late) + } + DefUse::Use | DefUse::NonUse => {} + }) + .visit_terminator(terminator, location); + + // Move arguments to a call are treated specially: the place that they + // represent is passed directly to the callee, which means that they are + // not allowed to alias any other move operand or the destination place. + // This is represented here by extending their live range to the late + // part, making it overlap with that of the destination place. + // + // Notably, this *doesn't* apply to TailCall. + if let mir::TerminatorKind::Call { + func: _, + args, + destination: _, + target: _, + unwind: _, + call_source: _, + fn_span: _, + } = &terminator.kind + { + for arg in args { + if let mir::Operand::Move(place) = arg.node { + builder.gen_(place.local, point, SplitPointEffect::Late); + builder.kill(place.local, point, SplitPointEffect::Late); + } + } + } + + // End the lifetimes of all locals at the end of the block. Successor + // blocks (which may not be continuous in the index space!) will + // initialize the lifetimes again from their entry state. + builder.kill_all(point, SplitPointEffect::Late); + } + + builder.matrix +} + +pub fn dump_liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: &'static str, + points: &DenseLocationMap, + matrix: &SparseIntervalMatrix, +) { + let locals_live_at = |split_point| { + matrix.rows().filter(|&r| matrix.contains(r, split_point)).collect::>() + }; + + if let Some(dumper) = MirDumper::new(tcx, pass_name, body) { + let extra_data = &|pass_where, w: &mut dyn std::io::Write| { + if let PassWhere::BeforeLocation(loc) = pass_where { + let point = points.point_from_location(loc); + let split_point = SplitPointIndex::new(point, SplitPointEffect::Early); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-early => {live:?}")?; + let split_point = SplitPointIndex::new(point, SplitPointEffect::Late); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-late => {live:?}")?; + } + Ok(()) + }; + + dumper.set_extra_data(extra_data).dump_mir(body) + } +} diff --git a/compiler/rustc_mir_dataflow/src/points.rs b/compiler/rustc_mir_dataflow/src/points.rs index 8568f325f1306..a4e5a3a79f04a 100644 --- a/compiler/rustc_mir_dataflow/src/points.rs +++ b/compiler/rustc_mir_dataflow/src/points.rs @@ -61,6 +61,15 @@ impl DenseLocationMap { PointIndex::new(start_index) } + /// Returns the `PointIndex` for the terminator in the given `BasicBlock`. O(1). + #[inline] + pub fn terminator(&self, block: BasicBlock) -> PointIndex { + let next_block = BasicBlock::new(block.index() + 1); + let next_start_index = + *self.statements_before_block.get(next_block).unwrap_or(&self.num_points); + PointIndex::new(next_start_index - 1) + } + /// Return the PointIndex for the block start of this index. #[inline] pub fn to_block_start(&self, index: PointIndex) -> PointIndex { diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 924125404a07a..14c178fec8ac5 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -155,7 +155,9 @@ pub(super) struct DestinationPropagation; impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { fn policy(&self, sess: &rustc_session::Session) -> PassPolicy { - PassPolicy::optimization(sess.mir_opt_level() >= 2) + PassPolicy::optimization( + sess.mir_opt_level() >= 2 && !sess.opts.unstable_opts.mir_move_elimination, + ) } #[tracing::instrument(level = "trace", skip(self, tcx, body))] diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index d2dd77c986318..0571b8f090925 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -165,6 +165,7 @@ declare_passes! { mod lower_slice_len : LowerSliceLenCalls; mod match_branches : MatchBranchSimplification; mod mentioned_items : MentionedItems; + mod move_elimination : MoveElimination; mod multiple_return_terminators : MultipleReturnTerminators; mod post_drop_elaboration : CheckLiveDrops; mod prettify : ReorderBasicBlocks, ReorderLocals; @@ -206,6 +207,7 @@ declare_passes! { mod sroa : ScalarReplacementOfAggregates; mod strip_debuginfo : StripDebugInfo; mod ssa_range_prop: SsaRangePropagation; + mod tail_copy_to_move : TailCopyToMove; mod unreachable_enum_branching : UnreachableEnumBranching; mod unreachable_prop : UnreachablePropagation; mod validate : Validator; @@ -761,6 +763,8 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' ©_prop::CopyProp, &dead_store_elimination::DeadStoreElimination::Final, &dest_prop::DestinationPropagation, + &tail_copy_to_move::TailCopyToMove, + &move_elimination::MoveElimination, &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, &large_enums::EnumSizeOpt { discrepancy: 128 }, diff --git a/compiler/rustc_mir_transform/src/move_elimination.rs b/compiler/rustc_mir_transform/src/move_elimination.rs new file mode 100644 index 0000000000000..2dca2cba88436 --- /dev/null +++ b/compiler/rustc_mir_transform/src/move_elimination.rs @@ -0,0 +1,1246 @@ +//! Eliminates copies and moves by unifying MIR places whose allocation ranges +//! are disjoint. +//! +//! See RFC 3943 for the local-lifetime semantics that make this optimization +//! possible. +//! +//! # Motivation +//! +//! MIR building can insert a lot of redundant copies, and Rust code in general +//! often tends to move values around a lot. The result is a lot of assignments +//! of the form `dest = {move} src;` in MIR. MIR building for constants in +//! particular tends to create additional locals that are only used inside a +//! single block to shuffle a value around unnecessarily. +//! +//! Additionally, Rust constructs nested aggregates by repeatedly moving values +//! into fields. For example, a function may build an inner value in a local, +//! move it into an outer aggregate, then move that aggregate into the caller's +//! destination. If these intermediate source and destination places have +//! different addresses, each layer needs an actual copy or move of the bytes. +//! +//! LLVM cannot remove these copies when both the source and destination +//! addresses are observed because merging the allocations would be an +//! observable change: the program could see that two addresses which were +//! previously distinct have become the same. This pass removes the copies +//! earlier, while MIR still has the information needed to prove that the two +//! allocation ranges do not overlap. +//! +//! # Optimization +//! +//! The basis of this optimization is place unification. If the source and +//! destination of an assignment have the same address, then the assignment is a +//! no-op. The same idea applies to aggregate construction: if a field operand +//! is already located at the corresponding field of the destination, then the +//! aggregate assignment does not need to copy those fields. +//! +//! The pass represents each unification as a mapping from a local to the place +//! that should replace it. Mappings are transitive, so `_3` can be resolved +//! through `_2.1` to `_1.0.1` if earlier mappings established those +//! relationships. +//! +//! The mapping is built by scanning the MIR for assignment statements. For +//! simple `Use` assignments, it tries to unify the source and destination +//! places. For `Aggregate` assignments, it tries to map each field operand to +//! the corresponding field in the assignment destination. Once all mappings +//! have been chosen, they are applied with one rewrite pass over the body. +//! +//! # Constraints +//! +//! Adding a mapping must preserve these conditions: +//! +//! * At least one side of the candidate pair must be a bare local. The pass can +//! map a local to a place with projections, but it cannot map between two +//! places that both already have projections. +//! +//! * Any projections in the mapped place must be stable everywhere the local is +//! used. `Deref` and `Index` projections are rejected because they may refer +//! to different memory at different points in the function. +//! +//! * The allocation ranges of the source and destination places must not +//! overlap. This is checked using `PreciseLiveness`, which computes the +//! points where each local must have a distinct allocation. The non-overlap +//! proof is required so that the operational semantics can allow both places +//! to have the same address. +//! +//! * Special-use locals such as arguments and the return place must keep their +//! roles. Temps may be mapped into an argument or return place, but two +//! special-use locals are not mapped into each other. +//! +//! * Some locals are used in contexts where projections cannot be added, such +//! as `Index` projections. These locals may only be replaced by another bare +//! local. +//! +//! # Storage reconstruction +//! +//! The original `StorageLive` and `StorageDead` statements no longer describe +//! the merged liveness produced by unification, so they are removed and rebuilt +//! from the liveness matrix when lifetime markers are emitted. This is done for +//! all locals, even ones that have not been merged, which has the additional +//! benefit of tightening the storage lifetime passed to LLVM. +//! +//! # Aliasing fixup +//! +//! MIR assignments currently require source and destination places not to +//! overlap for types that are not treated as scalars in codegen. After local +//! unification, some assignments may violate that invariant, so a final phase +//! rewrites them into a form codegen can handle. For each assignment: +//! +//! * Self-assignments, such as `_1 = _1`, are deleted. +//! +//! * Simple `Use` assignments whose source and destination overlap but are not +//! identical are routed through a temporary: the source is read into the +//! temporary first, and the temporary is then moved into the destination. +//! +//! * Aggregate assignments with any operand that aliases the destination are +//! decomposed into per-field assignments. Field self-assignments are dropped. +//! Other aliasing fields are read into temporaries first, then all +//! destination fields are written. For enum aggregates the discriminant is +//! set after fields are written. +//! +//! * Other rvalues, such as `Repeat` and `Cast`, are hoisted into a temporary +//! if any place they access aliases the destination. +//! +//! * Rvalues that operate only on scalar types, such as binary and unary ops, +//! `Discriminant`, `Ref`, and `RawPtr`, are left untouched because their +//! codegen does not rely on the no-aliasing assumption. + +use rustc_abi::{ExternAbi, FieldIdx, VariantIdx}; +use rustc_const_eval::util::most_packed_projection; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::thin_vec::ThinVec; +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, VisitPlacesWith, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_mir_dataflow::impls::{ + DefUse, SplitPointEffect, SplitPointIndex, dump_liveness_matrix, liveness_matrix, +}; +use rustc_mir_dataflow::points::DenseLocationMap; +use rustc_mir_dataflow::{Analysis, Backward, GenKill, ResultsVisitor, visit_results}; +use tracing::{debug, trace}; + +use crate::PassPolicy; +use crate::patch::MirPatch; + +pub(super) struct MoveElimination; + +impl<'tcx> crate::MirPass<'tcx> for MoveElimination { + fn policy(&self, sess: &rustc_session::Session) -> PassPolicy { + PassPolicy::optimization( + sess.mir_opt_level() >= 2 && sess.opts.unstable_opts.mir_move_elimination, + ) + } + + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let def_id = body.source.def_id(); + trace!(?def_id); + + let points = DenseLocationMap::new(body); + let mut liveness_matrix = + liveness_matrix(tcx, body, &points, Some("MoveElimination.liveness")); + + dump_liveness_matrix(tcx, body, "MoveElimination.pre-liveness", &points, &liveness_matrix); + + let unprojectable_locals = UnprojectableLocals::find(body); + trace!(?unprojectable_locals); + + let rust_call_tuples = find_rust_call_tuples(tcx, body); + trace!(?rust_call_tuples); + + let remapped_locals = PlaceUnification::run( + tcx, + body, + &mut liveness_matrix, + unprojectable_locals, + rust_call_tuples, + ); + + apply_mappings(tcx, body, &remapped_locals); + + dump_liveness_matrix(tcx, body, "MoveElimination.post-liveness", &points, &liveness_matrix); + + if tcx.sess.emit_lifetime_markers() { + reconstruct_storage(tcx, body, &points, &liveness_matrix); + } + + apply_alias_fixup(tcx, body); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Unprojectable locals + +/// Set of locals which can only be replaced with another local, instead of +/// an arbitrary place. This is usually because it is used directly as a +/// `Local` outside of a place (e.g. `Index` projections). +#[derive(Debug)] +struct UnprojectableLocals { + locals: DenseBitSet, +} + +impl UnprojectableLocals { + fn find(body: &Body<'_>) -> DenseBitSet { + let mut out = Self { locals: DenseBitSet::new_empty(body.local_decls.len()) }; + + // Arguments and return places have fixed roles and cannot be replaced + // with projected locals. + out.locals.insert(RETURN_PLACE); + for arg in body.args_iter() { + out.locals.insert(arg); + } + + out.visit_body(body); + out.locals + } +} + +impl<'tcx> Visitor<'tcx> for UnprojectableLocals { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + // We can't add more projections before a first position Deref projection. + if place.is_indirect() { + trace!( + "unprojectable local {:?} due to use as deref base at {location:?}", + place.local + ); + self.locals.insert(place.local); + } + + // Only call visit_local for projections, not the base local. + self.visit_projection(place.as_ref(), context, location); + } + + fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { + // Ignore uses in storage statements, we're going to remove all of those + // anyways. + if let PlaceContext::NonUse(NonUseContext::StorageLive | NonUseContext::StorageDead) = + context + { + return; + } + + // If this is reached, it means that this is a bare local used outside + // of a place, which means it cannot be replaced with a projection of + // another local. + trace!("unprojectable local {local:?} at {location:?} ({context:?})"); + self.locals.insert(local); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// "rust-call" tuple handling + +/// Search for tuple locals passed to calls using the "rust-call" ABI. +/// +/// For rust-call ABI calls, caller-side MIR passes the logical arguments as a +/// tuple operand. We want to avoid remapping other locals into fields of that +/// tuple, especially if one of those locals is borrowed. +/// +/// Since the tuple itself is never borrowed, it is trivial for LLVM alias +/// analysis to see that accesses to one argument do not affect the others, but +/// merging the arguments into tuple fields from the start can hide that +/// independence. +fn find_rust_call_tuples<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> DenseBitSet { + let mut rust_call_tuples = DenseBitSet::new_empty(body.local_decls.len()); + + for block in body.basic_blocks.iter() { + let terminator = block.terminator(); + let (func, args) = match &terminator.kind { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => (func, args), + _ => continue, + }; + + let sig = func.ty(&body.local_decls, tcx).fn_sig(tcx); + if sig.abi() != ExternAbi::RustCall { + continue; + } + + let arg_tuple = args.last().expect("rust-call ABI requires a tuple argument"); + let (Operand::Copy(place) | Operand::Move(place)) = arg_tuple.node else { + continue; + }; + if let Some(local) = place.as_local() { + rust_call_tuples.insert(local); + } + } + + rust_call_tuples +} + +//////////////////////////////////////////////////////////////////////////////// +// Local unification + +struct PlaceUnification<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, + liveness_matrix: &'a mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + remapped_locals: IndexVec>>, +} + +impl<'tcx> PlaceUnification<'_, 'tcx> { + fn run( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + liveness_matrix: &mut SparseIntervalMatrix, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, + ) -> IndexVec>> { + let mut visitor = PlaceUnification { + tcx, + body, + liveness_matrix, + unprojectable_locals, + rust_call_tuples, + remapped_locals: IndexVec::from_elem_n(None, body.local_decls.len()), + }; + visitor.visit_body(body); + + // Finalize the mappings by transitively resolving all locals to their + // new final place. + for local in visitor.remapped_locals.indices() { + if let Some(place) = visitor.remapped_locals[local] { + let place = visitor.resolve_place(place); + visitor.remapped_locals[local] = Some(place); + trace!("Remapped {local:?} to {place:?}"); + } + } + + visitor.remapped_locals + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn resolve_place(&self, mut place: Place<'tcx>) -> Place<'tcx> { + while let Some(new_place) = self.remapped_locals[place.local] { + place = new_place.project_deeper(place.projection, self.tcx); + } + place + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn can_unify_places(&self, a: Place<'tcx>, b: Place<'tcx>) -> Option<(Local, Place<'tcx>)> { + let a = self.resolve_place(a); + let b = self.resolve_place(b); + + if a.local == b.local { + if a.projection != b.projection { + trace!("cannot unify same local with different projections"); + } + return None; + } + + if self.rust_call_tuples.contains(a.local) || self.rust_call_tuples.contains(b.local) { + trace!("cannot unify {a:?} and {b:?} involving a rust-call tuple argument"); + return None; + } + + let (local, place) = match (a.as_local(), b.as_local()) { + (None, None) => { + trace!("cannot unify 2 places that both have projections"); + return None; + } + (None, Some(b)) => { + if self.unprojectable_locals.contains(b) { + trace!("cannot unify {b:?} which cannot be projected"); + return None; + } + (b, a) + } + (Some(a), None) => { + if self.unprojectable_locals.contains(a) { + trace!("cannot unify {a:?} which cannot be projected"); + return None; + } + (a, b) + } + (Some(a), Some(b)) => match (self.body.local_kind(a), self.body.local_kind(b)) { + ( + LocalKind::Arg | LocalKind::ReturnPointer, + LocalKind::Arg | LocalKind::ReturnPointer, + ) => { + trace!("cannot unify {a:?} and {b:?} which are both arguments or return place"); + return None; + } + (LocalKind::Arg | LocalKind::ReturnPointer, LocalKind::Temp) => (b, a.into()), + (LocalKind::Temp, _) => (a, b.into()), + }, + }; + + if most_packed_projection(self.tcx, &self.body.local_decls, place).is_some() { + trace!("cannot unify {place:?} which has packed field projections"); + return None; + } + + if !self.liveness_matrix.disjoint_rows(local, place.local) { + trace!("cannot unify {a:?} and {b:?} which have overlapping live ranges"); + return None; + } + + // FIXME(#112651): This can be removed afterwards. + let local_ty = self.body.local_decls[local].ty; + let place_ty = place.ty(&self.body.local_decls, self.tcx).ty; + if local_ty != place_ty { + trace!( + "cannot unify {a:?} and {b:?} which have different types due to subtyping ({local_ty:?} vs {place_ty:?})" + ); + return None; + } + + Some((local, place)) + } + + #[tracing::instrument(level = "trace", skip(self))] + fn remap_local(&mut self, local: Local, place: Place<'tcx>) { + self.remapped_locals[local] = Some(place); + + self.liveness_matrix.union_rows(local, place.local); + self.liveness_matrix.clear_row(local); + + // If the original local was unprojectable then this now also applies to + // the mapped local. + if self.unprojectable_locals.contains(local) { + debug_assert!(place.projection.is_empty()); + self.unprojectable_locals.insert(place.local); + } + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + location: Location, + ) { + // Attempt to unify each field operand with the corresponding field in + // the destination place. + let mut candidates = vec![]; + for (idx, operand) in operands.iter_enumerated() { + let (Operand::Copy(src) | Operand::Move(src)) = *operand else { + continue; + }; + let Some(src) = src.as_local() else { + continue; + }; + let dest = project_field(self.tcx, dest, idx, self.body.local_decls[src].ty); + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(dest, src.into()) { + candidates.push((local, place)); + } + } + + // Do the actual remapping *after* checking for live range overlaps. + // This is necessary because the input operands necessarily have + // overlapping live ranges. + for (local, place) in candidates { + self.remap_local(local, place); + } + } +} + +/// Since we are replacing all uses of a local with another place, we need to +/// ensure that the projections on that place are stable no matter where it is +/// used in the body. Additional this local may be used in debuginfo, so ensure +/// that the projections are compatible with usage in debuginfo. +fn check_projections(place: Place<'_>) -> bool { + place.projection.iter().all(|elem| elem.is_stable_offset() && elem.can_use_in_debuginfo()) +} + +impl<'tcx> Visitor<'tcx> for PlaceUnification<'_, 'tcx> { + fn visit_assign(&mut self, dest: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + if !check_projections(*dest) { + return; + } + match rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), _) => { + if !check_projections(*src) { + return; + } + + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(*src, *dest) { + self.remap_local(local, place); + } + } + Rvalue::Aggregate(aggregate_kind, operands) => match *aggregate_kind { + AggregateKind::Array(_) => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + location, + ), + AggregateKind::Tuple => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + location, + ), + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + location, + ) + } + AggregateKind::Adt(adt_did, var_idx, _, _, None) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + location, + ) + } + _ => {} + }, + _ => {} + }; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Apply place mappings to the MIR body. + +fn apply_mappings<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + remapped_locals: &IndexVec>>, +) { + let mut rewriter = PlaceUpdater { tcx, remapped_locals }; + rewriter.visit_body_preserves_cfg(body); +} + +struct PlaceUpdater<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + remapped_locals: &'a IndexVec>>, +} + +impl<'tcx> MutVisitor<'tcx> for PlaceUpdater<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[*local] { + trace!("replacing {local:?} with {new_place:?} at {location:?} ({context:?})"); + *local = new_place.as_local().expect("mapped place shouldn't have projections"); + } + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[place.local] { + trace!("replacing {place:?} with {new_place:?} at {location:?} ({context:?})"); + *place = new_place.project_deeper(place.projection, self.tcx) + } + + // Only call visit_local for projections, not the base local. + if let Some(new_projection) = self.process_projection(&place.projection, location) { + place.projection = self.tcx().mk_place_elems(&new_projection); + } + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + match statement.kind { + // Remove *all* storage statements. These are rebuilt from liveness + // information later. Also, since we've preserved StorageDead in + // unwind paths until now, we will want to remove those since they + // hurt LLVM's codegen. + StatementKind::StorageDead(_) | StatementKind::StorageLive(_) => { + statement.make_nop(true); + return; + } + _ => {} + } + + self.super_statement(statement, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Storage reconstruction + +/// Backward dataflow analysis which answers the question: from this point, is +/// there a path whose first access to a local is an initialization? +/// +/// This is used when a local is dead in a predecessor but maybe-live in its +/// successor. A `StorageLive` is inserted on that edge only if some continuation +/// initializes the local before reading it. If every continuation instead +/// reads the local first or never accesses it, `StorageLive` would be +/// unnecessary: it only allocates uninitialized storage and cannot make the +/// read valid. +/// +/// For example: +/// +/// ```text +/// bb1 bb2 +/// StorageLive(_1); _1 = ... // _1 is dead +/// _flag = true _flag = false +/// \ / +/// \ / +/// bb3 +/// switchInt(_flag) +/// / \ +/// bb4: use(_1) bb5: no use +/// ``` +/// +/// Liveness is path-insensitive, so `_1` is maybe-live in `bb3`: it is live in +/// `bb1` and `bb4`, but dead in `bb2` and `bb5`. Nevertheless, no `StorageLive` +/// is needed on `bb2 -> bb3`. The continuation to `bb5` never accesses `_1`, +/// while the continuation to `bb4` reads `_1` without initializing it first and +/// is therefore already UB. (The `_flag` assignments make that latter +/// continuation dynamically impossible, but this analysis does not need to +/// prove the correlation.) +/// +/// Live ranges which start in the middle of a block do not need this analysis: +/// such ranges always start at an initialization, so a `StorageLive` is +/// unconditionally required there. +/// +/// This analysis deliberately ignores the reconstructed `StorageDead` +/// boundaries. This can cause an initialization from a later allocation range +/// to propagate into an earlier range and result in an unnecessary +/// `StorageLive`, but cannot cause a required `StorageLive` to be omitted. +struct InitializedBeforeUse; + +impl<'tcx> Analysis<'tcx> for InitializedBeforeUse { + type Domain = DenseBitSet; + type Direction = Backward; + + const NAME: &'static str = "initialized-before-use"; + + fn bottom_value(&self, body: &Body<'tcx>) -> Self::Domain { + DenseBitSet::new_empty(body.local_decls.len()) + } + + fn initialize_start_block(&self, _body: &Body<'tcx>, _state: &mut Self::Domain) {} + + fn apply_primary_statement_effect( + &self, + state: &mut Self::Domain, + statement: &Statement<'tcx>, + location: Location, + ) { + // In backward order, process writes before reads. + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Def | DefUse::PartialWrite) { + state.gen_(place.local); + } + }) + .visit_statement(statement, location); + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Use) { + state.kill(place.local); + } + }) + .visit_statement(statement, location); + } + + fn apply_primary_terminator_effect( + &self, + state: &mut Self::Domain, + terminator: &Terminator<'tcx>, + location: Location, + ) { + // In backward order, process writes before reads. + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Def | DefUse::PartialWrite) { + state.gen_(place.local); + } + }) + .visit_terminator(terminator, location); + VisitPlacesWith(|place: Place<'tcx>, context| { + if matches!(DefUse::for_place(place, context), DefUse::Use) { + state.kill(place.local); + } + }) + .visit_terminator(terminator, location); + } +} + +impl InitializedBeforeUse { + /// Computes the analysis state at the start of each block. + fn compute<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + ) -> IndexVec> { + let results = + Self.iterate_to_fixpoint(tcx, body, Some("MoveElimination.initialized-before-use")); + let mut block_start = IndexVec::from_elem_n( + DenseBitSet::new_empty(body.local_decls.len()), + body.basic_blocks.len(), + ); + + struct BlockStartVisitor<'a> { + block_start: &'a mut IndexVec>, + } + + impl<'tcx> ResultsVisitor<'tcx, InitializedBeforeUse> for BlockStartVisitor<'_> { + fn visit_block_exit(&mut self, state: &DenseBitSet, block: BasicBlock) { + self.block_start[block].clone_from(state); + } + } + + visit_results( + body, + rustc_middle::mir::traversal::reachable(body).map(|(block, _)| block), + &results, + &mut BlockStartVisitor { block_start: &mut block_start }, + ); + block_start + } +} + +/// Helper function to split a critical edge if necessary. +fn get_or_split_edge<'tcx>( + patcher: &mut MirPatch<'tcx>, + body: &Body<'tcx>, + split_edges: &mut FxHashMap<(BasicBlock, BasicBlock), BasicBlock>, + pred: BasicBlock, + succ: BasicBlock, +) -> BasicBlock { + if let Some(&split_bb) = split_edges.get(&(pred, succ)) { + return split_bb; + } + let source_info = body.basic_blocks[pred].terminator().source_info; + let split_bb = patcher.new_block(BasicBlockData::new( + Some(Terminator { + source_info, + kind: TerminatorKind::Goto { target: succ }, + attributes: ThinVec::new(), + }), + body.basic_blocks[succ].is_cleanup, + )); + patcher.mutate_terminator(body, pred, |kind| { + kind.successors_mut(|t| { + if *t == succ { + *t = split_bb; + } + }); + }); + split_edges.insert((pred, succ), split_bb); + split_bb +} + +/// Don't insert `StorageDead` statements in cleanup blocks and unreachable blocks. +fn should_insert_storage_dead<'tcx>(block_data: &BasicBlockData<'tcx>) -> bool { + !block_data.is_cleanup && !matches!(block_data.terminator().kind, TerminatorKind::Unreachable) +} + +/// Re-constructs storage statements for all locals. +fn reconstruct_storage<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + points: &DenseLocationMap, + liveness_matrix: &SparseIntervalMatrix, +) { + let initialized_before_use = InitializedBeforeUse::compute(tcx, body); + let mut patcher = MirPatch::new(body); + let mut split_edges: FxHashMap<(BasicBlock, BasicBlock), BasicBlock> = Default::default(); + let mut storage_lives = Vec::new(); + + for local in body.local_decls.indices() { + // Arguments and return values don't use storage statements. + match body.local_kind(local) { + LocalKind::Arg | LocalKind::ReturnPointer => continue, + LocalKind::Temp => {} + } + + // Ignore dead locals. + let Some(row) = liveness_matrix.row(local) else { continue }; + if row.is_empty() { + continue; + } + + let mut emit_storage_live_in_preds = + |body: &mut Body<'tcx>, + patcher: &mut MirPatch<'tcx>, + storage_lives: &mut Vec<(Location, Local)>, + local: Local, + block: BasicBlock| { + if !initialized_before_use[block].contains(local) { + // No continuation initializes the local before reading it, + // so allocating storage cannot make any such read valid. + return; + } + + for &pred in &body.basic_blocks.predecessors()[block].clone() { + // If the local is live at any point in the predecessor's + // terminator then no StorageLive is needed. + let term = points.terminator(pred); + let term_early = SplitPointIndex::new(term, SplitPointEffect::Early); + let term_late = SplitPointIndex::new(term, SplitPointEffect::Late); + if !row.intersects_range(term_early..=term_late) { + // The local must be live on at least one predecessor, + // so if this is the only one then there is nothing to + // do. + debug_assert!(body.basic_blocks.predecessors()[block].len() > 1); + + // If the predecessor block has multiple successors then + // we need to split the critical edge before inserting + // StorageLive, otherwise the local would end up live on + // paths where it is supposed to be dead. + let loc = if body.basic_blocks[pred].terminator().successors().count() > 1 { + get_or_split_edge(patcher, body, &mut split_edges, pred, block) + .start_location() + } else { + body.terminator_loc(pred) + }; + storage_lives.push((loc, local)); + } + } + }; + let emit_storage_dead_in_succs = + |body: &mut Body<'tcx>, + patcher: &mut MirPatch<'tcx>, + local: Local, + block: BasicBlock| { + for succ in body.basic_blocks[block].terminator().successors() { + if !should_insert_storage_dead(&body.basic_blocks[succ]) { + continue; + } + + if !row.contains(SplitPointIndex::new( + points.entry_point(succ), + SplitPointEffect::Early, + )) { + // We don't care about critical edges here: if the local + // is already dead in the successor then it doesn't + // matter if we emit a redundant StorageDead. + + patcher.add_statement( + succ.start_location(), + StatementKind::StorageDead(local), + ); + } + } + }; + + // Iterate through the live range of the local and insert `StorageLive` + // and `StorageDead` at the points where it transitions from dead to + // live and vice versa. + // + // Note that the range here is an *inclusive range*. + for range in row.iter_intervals() { + let start = points.to_location(range.start.point()); + let end = points.to_location(range.last.point()); + + // If the live range starts at the `Early` point then it means that + // the value came from a predecessor block. A write from the first + // statement would happen at the `Late` point instead. + if range.start.effect() == SplitPointEffect::Early && start.statement_index == 0 { + // If the local is dead at the end of any predecessor block then + // emit a `StorageLive` before the terminator. + emit_storage_live_in_preds( + body, + &mut patcher, + &mut storage_lives, + local, + start.block, + ); + } else { + // Otherwise just add `StorageLive` before the statement that + // starts the live range. + storage_lives.push((start, local)); + } + + // The live range may span multiple blocks because + // `SparseIntervalMatrix` will coalesce adjacent ranges. If this + // happens then we need to repeat the start of block logic (see + // above) and end of block logic (see below) at each block boundary. + let mut current_block = start.block; + debug_assert!(start.block <= end.block); + while current_block != end.block { + if should_insert_storage_dead(&body.basic_blocks[current_block]) { + emit_storage_dead_in_succs(body, &mut patcher, local, current_block); + } + current_block = BasicBlock::from_usize(current_block.index() + 1); + emit_storage_live_in_preds( + body, + &mut patcher, + &mut storage_lives, + local, + current_block, + ); + } + + // We need to insert `StorageDead` after the last statement that + // uses a local. If this is a terminator then we need to instead + // insert it at the start of every successor block where the local + // is dead on entry. + if should_insert_storage_dead(&body.basic_blocks[end.block]) { + if range.last.point() == points.terminator(end.block) { + emit_storage_dead_in_succs(body, &mut patcher, local, current_block); + } else { + patcher.add_statement( + end.successor_within_block(), + StatementKind::StorageDead(local), + ); + } + } + } + } + + // Queue all `StorageLive` statements after `StorageDead` so that, when + // both are inserted at the same location, `StorageDead` always precedes + // `StorageLive` to avoid false overlaps. + for (loc, local) in storage_lives { + patcher.add_statement(loc, StatementKind::StorageLive(local)); + } + + patcher.apply(body); +} + +//////////////////////////////////////////////////////////////////////////////// +// Aliasing assignment fixup +// +// MIR assignments currently do not allow source and destination to alias, so +// fix this in post-processing. + +fn apply_alias_fixup<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut patcher = MirPatch::new(body); + let mut fixup = AliasFixup { tcx, local_decls: &body.local_decls, patcher: &mut patcher }; + for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + fixup.visit_basic_block_data(block, data); + } + patcher.apply(body); +} + +/// Returns whether 2 places alias, ignoring indirect places. +fn places_directly_alias<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &IndexVec>, + a: Place<'tcx>, + b: Place<'tcx>, +) -> bool { + // This function doesn't handle indirect aliasing. + if a.local != b.local || a.is_indirect_first_projection() || b.is_indirect_first_projection() { + return false; + } + + for ((prefix, elem_a), (_, elem_b)) in a.iter_projections().zip(b.iter_projections()) { + // Continue until we find the first mismatching projection. + if elem_a == elem_b { + continue; + } + + match (elem_a, elem_b) { + // Disjoint fields don't alias except if they are union fields. + (PlaceElem::Field(_, _), PlaceElem::Field(_, _)) => { + let ty = prefix.ty(local_decls, tcx).ty; + return ty.is_union(); + } + + // Disjoint slice elements don't alias. + ( + PlaceElem::ConstantIndex { offset: offset_a, min_length: _, from_end: from_end_a }, + PlaceElem::ConstantIndex { offset: offset_b, min_length: _, from_end: from_end_b }, + ) if from_end_a == from_end_b && offset_a != offset_b => { + return false; + } + + // Conservatively assume the places may alias. + _ => return true, + } + } + + // If the projections are identical *or* one is a prefix of the other then + // the places alias. + true +} + +struct AliasFixup<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + local_decls: &'a IndexVec>, + patcher: &'a mut MirPatch<'tcx>, +} + +impl<'tcx> AliasFixup<'_, 'tcx> { + fn isolate_rvalue_to_local( + &mut self, + rvalue: Rvalue<'tcx>, + source_info: SourceInfo, + location: Location, + ) -> Place<'tcx> { + let ty = rvalue.ty(self.local_decls, self.tcx); + let temp = Place::from(self.patcher.new_temp(ty, source_info.span)); + trace!("isolating {rvalue:?} to {temp:?} due to conflict"); + self.patcher.add_statement(location, StatementKind::StorageLive(temp.local)); + self.patcher.add_assign(location, Place::from(temp), rvalue); + self.patcher.add_statement( + location.successor_within_block(), + StatementKind::StorageDead(temp.local), + ); + temp + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + enum_variant: Option, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + source_info: SourceInfo, + location: Location, + ) { + // Fast path: if no direct operand aliases the destination, we're done. + // + // We only look for direct aliases here, which is sufficient because we + // know the input MIR did not have any aliasing and we didn't introduce + // any indirect aliasing in this pass. + // + // If the destination place is indirect then it cannot be the start of + // a lifetime as per the RFC 3943 MIR semantics. This means that the + // lifetime of the underlying allocation must have started earlier, + // which overlaps the early point of the assignment statement. Therefore + // we couldn't have unified any source operand with this destination + // place. + // + // If the source place is indirect then a similar reasoning applies. The + // only exception is if there are multiple source places (e.g. + // aggregates). In that situation it's possible for an indirect source + // to overlap the destination if and only if there is also a direct + // source that overlaps it: + // + // _2 = &_1 + // _3 = (copy *_2, move _1) // _1 becomes _3.1 after unification + // + // We handle this here in 2 ways: if there is no direct alias, then + // we're fine. Otherwise, treat all indirect sources as potentially + // aliasing with the destination operand. + let has_direct_alias = operands.iter().any(|op| match op { + Operand::Copy(src) | Operand::Move(src) => { + places_directly_alias(self.tcx, self.local_decls, dest, *src) + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => false, + }); + if !has_direct_alias { + return; + } + + debug!("splitting aggregate assignment at {location:?}"); + + // Split into per-field assignments. + let mut assignments = vec![]; + for (idx, op) in operands.iter_enumerated() { + let field_ty = op.ty(self.local_decls, self.tcx); + let dest_field = project_field(self.tcx, dest, idx, field_ty); + + let emit_op = match op { + Operand::Copy(src) | Operand::Move(src) => { + if *src == dest_field { + // Skip identity assignments. + continue; + } else if src.is_indirect_first_projection() + || places_directly_alias(self.tcx, self.local_decls, dest, *src) + { + // Partial alias: hoist the source to a temp first so the + // per-field write no longer overlaps the dest. Indirect + // sources also need hoisting here because they may point + // at one of the direct aliasing operands. + Operand::Move(self.isolate_rvalue_to_local( + Rvalue::Use(op.clone(), WithRetag::No), + source_info, + location, + )) + } else { + op.clone() + } + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => op.clone(), + }; + assignments.push((dest_field, emit_op)); + } + + // Perform assignments *after* all aliasing fields have been read into + // temporary locals. + for (dest_field, emit_op) in assignments { + self.patcher.add_assign(location, dest_field, Rvalue::Use(emit_op, WithRetag::No)); + } + + // Delete the original aggregate assignment. + self.patcher.nop_statement(location); + + // For enum variants, set the discriminant after all field writes. + if let Some(variant_index) = enum_variant { + self.patcher.add_statement( + location, + StatementKind::SetDiscriminant { place: Box::new(dest), variant_index }, + ); + } + } +} + +impl<'tcx> MutVisitor<'tcx> for AliasFixup<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + // Fixup the MIR to remove aliasing assignments. + if let StatementKind::Assign((dest, rvalue)) = &mut statement.kind { + match *rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), with_retag) => { + if places_directly_alias(self.tcx, self.local_decls, *dest, src) { + if src == *dest { + debug!("{:?} turned into self-assignment, deleting", location); + statement.make_nop(true); + } else { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), with_retag); + } + } + } + Rvalue::Aggregate(AggregateKind::Array(_), ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate(AggregateKind::Tuple, ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate( + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)), + ref mut operands, + ) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + None, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ) + } + Rvalue::Aggregate( + AggregateKind::Adt(adt_did, var_idx, _, _, None), + ref mut operands, + ) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + def.is_enum().then_some(var_idx), + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + statement.source_info, + location, + ) + } + + // For other rvalues, don't try to split them into components + // and instead just introduce a temporary if there is any + // aliasing + Rvalue::Aggregate(..) + | Rvalue::Repeat(..) + | Rvalue::Cast(..) + | Rvalue::CopyForDeref(..) + | Rvalue::WrapUnsafeBinder(..) => { + let mut overlaps_dest = false; + VisitPlacesWith(|place, _ctxt| { + if places_directly_alias(self.tcx, self.local_decls, *dest, place) { + overlaps_dest = true; + } + }) + .visit_rvalue(rvalue, location); + if overlaps_dest { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), WithRetag::No); + } + } + + // These either cannot have aliasing, or allow it because they + // only operate on scalar backend types. + Rvalue::Use(Operand::Constant(..) | Operand::RuntimeChecks(..), _) + | Rvalue::Ref(..) + | Rvalue::ThreadLocalRef(..) + | Rvalue::BinaryOp(..) + | Rvalue::UnaryOp(..) + | Rvalue::Discriminant(..) + | Rvalue::RawPtr(..) + | Rvalue::Reborrow(..) => {} + } + } + } +} diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index bd4cbcd89163c..15230c70f866e 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -215,6 +215,21 @@ impl<'tcx> MirPatch<'tcx> { self.term_patch_map.insert(block, new); } + /// Modifies the terminator of a block, reading the existing patch if one exists or + /// cloning from the body otherwise. + pub(crate) fn mutate_terminator( + &mut self, + body: &Body<'tcx>, + bb: BasicBlock, + f: impl FnOnce(&mut TerminatorKind<'tcx>), + ) { + let kind = self + .term_patch_map + .entry(bb) + .or_insert_with(|| body.basic_blocks[bb].terminator().kind.clone()); + f(kind); + } + /// Mark given statement to be replaced by a `Nop`. /// /// This method only works on statements from the initial body, and cannot be used to remove diff --git a/compiler/rustc_mir_transform/src/tail_copy_to_move.rs b/compiler/rustc_mir_transform/src/tail_copy_to_move.rs new file mode 100644 index 0000000000000..096ae20cc6ee7 --- /dev/null +++ b/compiler/rustc_mir_transform/src/tail_copy_to_move.rs @@ -0,0 +1,258 @@ +//! Rewrite final-use copies before return into moves. +//! +//! # The problem +//! +//! MIR building represents reads of values whose type is `Copy` using +//! `Operand::Copy`, including when such a local is returned. If that local's +//! address has ever been observed, then the local's allocation is semantically +//! valid until its `StorageDead` or function exit. This keeps the local live +//! across `_0 = copy local`, so its live range overlaps with the return place +//! and `MoveElimination` cannot unify the source local with `_0`. +//! +//! # The solution +//! +//! At function return, all local allocations are about to become invalid +//! anyway. After borrowck, this pass can therefore turn a final-use `Copy` into +//! a `Move`, as long as shortening the source local's live range has no +//! observable effect before the return happens. Concretely, between the +//! transformed copy (now a move) and the return, there may only be writes to +//! unborrowed locals, storage markers, nops, and gotos. +//! +//! # The algorithm +//! +//! Start from every `Return` terminator, with `_0` treated as used by the +//! return. Then scan predecessor blocks backward through `Goto` edges, forming +//! a return-tail tree. The scan maintains `used_after`, the set of locals +//! accessed later on that path. +//! +//! A `Copy` operand is rewritten to a `Move` when its base local is not in +//! `used_after`. Then any locals touched by that operand, including +//! index-projection locals, are added to `used_after` before the backward scan +//! continues. +//! +//! The scan stops when accessing an indirect place because that may access any +//! borrowed local, which would make the pass unable to prove any useful final +//! uses. It also stops at writes to borrowed locals, because those can create a +//! new address-observed allocation range whose overlap with an earlier borrowed +//! local must be preserved. + +use std::ops::ControlFlow; + +use rustc_index::bit_set::DenseBitSet; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; +use rustc_mir_dataflow::impls::borrowed_locals; + +use crate::PassPolicy; + +pub(super) struct TailCopyToMove; + +impl<'tcx> crate::MirPass<'tcx> for TailCopyToMove { + fn policy(&self, sess: &rustc_session::Session) -> PassPolicy { + PassPolicy::optimization( + sess.mir_opt_level() >= 2 && sess.opts.unstable_opts.mir_move_elimination, + ) + } + + #[tracing::instrument(level = "trace", skip(self, _tcx, body))] + fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let borrowed = borrowed_locals(body); + let predecessors = body.basic_blocks.predecessors().clone(); + let mut stack = Vec::new(); + + // A return terminator implicitly uses the return place. Walking + // backward through assignments records the locals accessed later on + // this path. + for (bb, data) in body.basic_blocks.iter_enumerated() { + if matches!(data.terminator().kind, TerminatorKind::Return) { + let mut used_after = DenseBitSet::new_empty(body.local_decls.len()); + used_after.insert(RETURN_PLACE); + stack.push(TailState { block: bb, used_after }); + } + } + + while let Some(mut state) = stack.pop() { + // `scan_block` rewrites final-use copies in this block and updates + // `used_after` to the locals whose allocation is accessed after the + // block starts. If the block is not pure tail code, this path is + // done. + if scan_block(body, state.block, &mut state.used_after, &borrowed).is_break() { + continue; + } + + // Continue through predecessor blocks only when the predecessor's + // terminator is a plain `Goto` to this block. Other terminators are + // control-flow or effect boundaries. + let mut first = None; + for pred in predecessors[state.block].iter().copied() { + let terminator = body.basic_blocks[pred].terminator(); + if let TerminatorKind::Goto { target } = terminator.kind { + debug_assert_eq!(target, state.block); + if first.is_none() { + first = Some(pred); + } else { + stack.push(TailState { block: pred, used_after: state.used_after.clone() }); + } + } + } + + // Avoid cloning the bitset for the first predecessor. + if let Some(pred) = first { + stack.push(TailState { block: pred, used_after: state.used_after }); + } + } + } +} + +struct TailState { + block: BasicBlock, + used_after: DenseBitSet, +} + +/// Scan a block backward while the return-tail invariant still holds. +/// +/// The invariant is that a whole-local `Copy` can be changed to a `Move` only +/// if this path has no later access to that local's allocation before +/// returning, and no later operation whose observable behavior could depend on +/// ending an address-observed local's allocation early. `used_after` tracks +/// those later local-allocation accesses. +fn scan_block<'tcx>( + body: &mut Body<'tcx>, + block: BasicBlock, + used_after: &mut DenseBitSet, + borrowed: &DenseBitSet, +) -> ControlFlow<()> { + for statement in body.basic_blocks.as_mut_preserves_cfg()[block].statements.iter_mut().rev() { + match &mut statement.kind { + // Under the local lifetime semantics from RFC 3943, `StorageLive` + // does not allocate, and `StorageDead` has no effect if the local + // was already freed by a move. These markers therefore do not + // affect whether a copy can be treated as a final use. + StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Nop => {} + StatementKind::Assign((place, rhs)) => { + // Accessing an indirect place may touch any borrowed local, so + // continuing would require treating all borrowed locals as used + // after this point. + if place.is_indirect_first_projection() { + return ControlFlow::Break(()); + } + + // Writing to a borrowed local can start a new allocation range. + // Shortening an earlier borrowed local could remove an overlap + // with that new range. + if borrowed.contains(place.local) { + return ControlFlow::Break(()); + } + + // A destination write accesses the base local, and evaluating + // the destination may also access projection locals, such as an + // index. + record_place_locals(*place, used_after); + + // This pass only models `Use` and `Aggregate` rvalues whose + // operands are direct. Other rvalues are outside the + // conservative return-tail shape handled here. + process_rvalue(rhs, used_after)?; + } + StatementKind::SetDiscriminant { place, .. } => { + // Accessing an indirect place may touch any borrowed local, so + // continuing would require treating all borrowed locals as used + // after this point. + if place.is_indirect_first_projection() { + return ControlFlow::Break(()); + } + + // Writing to a borrowed local can start a new allocation range. + // Shortening an earlier borrowed local could remove an overlap + // with that new range. + if borrowed.contains(place.local) { + return ControlFlow::Break(()); + } + + // `SetDiscriminant` has a validity invariant on the rest of the + // place, so treat the base local as accessed along with any + // projection locals. + record_place_locals(**place, used_after); + } + _ => { + // Anything else may perform effects or evaluate places in ways + // this pass does not model, so it is not part of the pure + // return tail. + return ControlFlow::Break(()); + } + } + } + + ControlFlow::Continue(()) +} + +/// Records all locals used in a place, including `Index` projections in +/// `used_after`. +fn record_place_locals<'tcx>(place: Place<'tcx>, used_after: &mut DenseBitSet) { + for local in place.as_ref().accessed_locals() { + used_after.insert(local); + } +} + +/// Process the RHS of an assignment in a pure return tail. +fn process_rvalue<'tcx>( + rvalue: &mut Rvalue<'tcx>, + used_after: &mut DenseBitSet, +) -> ControlFlow<()> { + match rvalue { + Rvalue::Use(operand, _) => process_operand(operand, used_after), + Rvalue::Aggregate(_, operands) => { + // Operands are evaluated left-to-right. We scan them right-to-left + // so `used_after` includes uses later in the same statement. If an + // operand accesses an indirect place, only earlier operands and + // earlier statements are outside the pure tail. + for operand in operands.iter_mut().rev() { + process_operand(operand, used_after)?; + } + + ControlFlow::Continue(()) + } + _ => { + // This pass doesn't model other rvalues, so they are not part of + // the pure return tail. + ControlFlow::Break(()) + } + } +} + +/// Process one operand in an rvalue. +fn process_operand<'tcx>( + operand: &mut Operand<'tcx>, + used_after: &mut DenseBitSet, +) -> ControlFlow<()> { + let place = match operand { + Operand::Copy(place) | Operand::Move(place) if place.is_indirect_first_projection() => { + // Accessing an indirect place may touch any borrowed local. + // Continuing would require treating all borrowed locals as used + // after this point, which would prevent the useful copy-to-move + // rewrites this pass is looking for. + return ControlFlow::Break(()); + } + Operand::Copy(place) => { + let place = *place; + // No later operation in the scanned tail accesses this local's + // allocation, so this copy is a final use on the current return + // path and can be represented as a move. + if !used_after.contains(place.local) { + *operand = Operand::Move(place); + } + Some(place) + } + Operand::Move(place) => Some(*place), + Operand::Constant(_) | Operand::RuntimeChecks(_) => None, + }; + + if let Some(place) = place { + // Reading an operand place accesses its base local, and evaluating its + // projections may access additional locals, such as the index local in + // `place[index]`. + record_place_locals(place, used_after); + } + + ControlFlow::Continue(()) +} diff --git a/compiler/rustc_public/src/mir/body.rs b/compiler/rustc_public/src/mir/body.rs index a65798aff1a00..f4e93bd05a20c 100644 --- a/compiler/rustc_public/src/mir/body.rs +++ b/compiler/rustc_public/src/mir/body.rs @@ -551,9 +551,6 @@ pub enum Rvalue { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(AggregateKind, Vec), /// * `Offset` has the same semantics as `<*const T>::offset`, except that the second diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 90459090ced87..3b2887f39b3ea 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2658,6 +2658,8 @@ options! { mir_include_spans: MirIncludeSpans = (MirIncludeSpans::default(), parse_mir_include_spans, [UNTRACKED], "include extra comments in mir pretty printing, like line numbers and statement indices, \ details about types, etc. (boolean for all passes, 'nll' to enable in NLL MIR only, default: 'nll')"), + mir_move_elimination: bool = (false, parse_bool, [TRACKED], + "enable the experimental MIR move elimination pass (default: no)"), mir_opt_bisect_limit: Option = (None, parse_opt_number, [TRACKED], "limit the number of MIR optimization pass executions (global across all bodies). \ Pass executions after this limit are skipped and reported. (default: no limit)"), diff --git a/src/tools/miri/src/intrinsics/mod.rs b/src/tools/miri/src/intrinsics/mod.rs index 7d7081fb609fb..1a302099ecac2 100644 --- a/src/tools/miri/src/intrinsics/mod.rs +++ b/src/tools/miri/src/intrinsics/mod.rs @@ -40,6 +40,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { &mut self, instance: ty::Instance<'tcx>, args: &[OpTy<'tcx>], + caller_moved_locals: &mut Vec, dest: &PlaceTy<'tcx>, ret: Option, unwind: mir::UnwindAction, @@ -53,7 +54,14 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let intrinsic_name = this.tcx.item_name(instance.def_id()); let intrinsic_name = intrinsic_name.as_str(); - let res = this.emulate_intrinsic_by_name(intrinsic_name, instance.args, args, dest, ret)?; + let res = this.emulate_intrinsic_by_name( + intrinsic_name, + instance.args, + args, + caller_moved_locals, + dest, + ret, + )?; res.jump_to_next_block(this, dest, ret, Some(unwind), |this| { // We haven't handled the intrinsic, let's see if we can use a fallback body. if this.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden { @@ -84,6 +92,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { intrinsic_name: &str, generic_args: ty::GenericArgsRef<'tcx>, args: &[OpTy<'tcx>], + caller_moved_locals: &mut Vec, dest: &PlaceTy<'tcx>, ret: Option, ) -> InterpResult<'tcx, EmulateItemResult> { @@ -102,7 +111,14 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "catch_unwind" => { let [try_fn, data, catch_fn] = check_intrinsic_arg_count(args)?; - this.handle_catch_unwind(try_fn, data, catch_fn, dest, ret)?; + this.handle_catch_unwind( + try_fn, + data, + catch_fn, + std::mem::take(caller_moved_locals), + dest, + ret, + )?; // This pushed a stack frame, don't jump to `ret`. return interp_ok(EmulateItemResult::AlreadyJumped); } diff --git a/src/tools/miri/src/machine.rs b/src/tools/miri/src/machine.rs index 361ebef73b357..0d95cd9d2282f 100644 --- a/src/tools/miri/src/machine.rs +++ b/src/tools/miri/src/machine.rs @@ -1290,11 +1290,12 @@ impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> { ecx: &mut MiriInterpCx<'tcx>, instance: ty::Instance<'tcx>, args: &[OpTy<'tcx>], + caller_moved_locals: &mut Vec, dest: &PlaceTy<'tcx>, ret: Option, unwind: mir::UnwindAction, ) -> InterpResult<'tcx, Option>> { - ecx.call_intrinsic(instance, args, dest, ret, unwind) + ecx.call_intrinsic(instance, args, caller_moved_locals, dest, ret, unwind) } #[inline(always)] @@ -1333,7 +1334,11 @@ impl<'tcx> Machine<'tcx> for MiriMachine<'tcx> { ExternAbi::Rust, &[], None, - ReturnContinuation::Goto { ret: None, unwind: mir::UnwindAction::Unreachable }, + ReturnContinuation::Goto { + ret: None, + unwind: mir::UnwindAction::Unreachable, + caller_moved_locals: vec![], + }, )?; interp_ok(()) } diff --git a/src/tools/miri/src/shims/panic.rs b/src/tools/miri/src/shims/panic.rs index 50e32cffaee3f..55f89f3525e61 100644 --- a/src/tools/miri/src/shims/panic.rs +++ b/src/tools/miri/src/shims/panic.rs @@ -22,7 +22,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ExternAbi::Rust, &[this.mplace_to_imm_ptr(&msg, None)?], None, - ReturnContinuation::Goto { ret: None, unwind }, + ReturnContinuation::Goto { ret: None, unwind, caller_moved_locals: vec![] }, ) } @@ -41,7 +41,11 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ExternAbi::Rust, &[this.mplace_to_imm_ptr(&msg, None)?], None, - ReturnContinuation::Goto { ret: None, unwind: mir::UnwindAction::Unreachable }, + ReturnContinuation::Goto { + ret: None, + unwind: mir::UnwindAction::Unreachable, + caller_moved_locals: vec![], + }, ) } @@ -58,9 +62,11 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Forward to `panic_bounds_check` lang item. // First arg: index. - let index = this.read_immediate(&this.eval_operand(index, None)?)?; + let index = this.eval_operand(index, None)?; + let index = this.read_immediate(&index)?; // Second arg: len. - let len = this.read_immediate(&this.eval_operand(len, None)?)?; + let len = this.eval_operand(len, None)?; + let len = this.read_immediate(&len)?; // Call the lang item. let panic_bounds_check = this.tcx.lang_items().panic_bounds_check_fn().unwrap(); @@ -70,16 +76,18 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ExternAbi::Rust, &[index, len], None, - ReturnContinuation::Goto { ret: None, unwind }, + ReturnContinuation::Goto { ret: None, unwind, caller_moved_locals: vec![] }, )?; } MisalignedPointerDereference { required, found } => { // Forward to `panic_misaligned_pointer_dereference` lang item. // First arg: required. - let required = this.read_immediate(&this.eval_operand(required, None)?)?; + let required = this.eval_operand(required, None)?; + let required = this.read_immediate(&required)?; // Second arg: found. - let found = this.read_immediate(&this.eval_operand(found, None)?)?; + let found = this.eval_operand(found, None)?; + let found = this.read_immediate(&found)?; // Call the lang item. let panic_misaligned_pointer_dereference = @@ -91,7 +99,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ExternAbi::Rust, &[required, found], None, - ReturnContinuation::Goto { ret: None, unwind }, + ReturnContinuation::Goto { ret: None, unwind, caller_moved_locals: vec![] }, )?; } @@ -104,7 +112,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ExternAbi::Rust, &[], None, - ReturnContinuation::Goto { ret: None, unwind }, + ReturnContinuation::Goto { ret: None, unwind, caller_moved_locals: vec![] }, )?; } } diff --git a/src/tools/miri/src/shims/unwind.rs b/src/tools/miri/src/shims/unwind.rs index 820a78725eedc..975104de80ce0 100644 --- a/src/tools/miri/src/shims/unwind.rs +++ b/src/tools/miri/src/shims/unwind.rs @@ -62,6 +62,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { try_fn: &OpTy<'tcx>, data: &OpTy<'tcx>, catch_fn: &OpTy<'tcx>, + caller_moved_locals: Vec, dest: &PlaceTy<'tcx>, ret: Option, ) -> InterpResult<'tcx> { @@ -94,7 +95,11 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { &[data.clone()], None, // Directly return to caller. - ReturnContinuation::Goto { ret, unwind: mir::UnwindAction::Continue }, + ReturnContinuation::Goto { + ret, + unwind: mir::UnwindAction::Continue, + caller_moved_locals, + }, )?; // We ourselves will return `0`, eventually (will be overwritten if we catch a panic). @@ -149,6 +154,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { ret: catch_unwind.ret, // `catch_fn` must not unwind. unwind: mir::UnwindAction::Unreachable, + caller_moved_locals: vec![], }, )?; diff --git a/src/tools/miri/tests/fail/move_elimination/address_after_call_move.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.move_elimination.stderr new file mode 100644 index 0000000000000..79bc3fe41caa9 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/address_after_call_move.rs:LL:CC + | +LL | ptr = &raw const value; + | ^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/address_after_call_move.rs b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.rs new file mode 100644 index 0000000000000..9a9feafba30ad --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/address_after_call_move.rs @@ -0,0 +1,26 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: (u8, u8); + let unit: (); + let ptr: *const (u8, u8); + { + value = (1, 2); + Call(unit = consume(Move(value)), ReturnTo(after_call), UnwindContinue()) + } + after_call = { + ptr = &raw const value; //~[move_elimination] ERROR: live but unallocated + Return() + } + } +} + +fn consume(_: (u8, u8)) {} diff --git a/src/tools/miri/tests/fail/move_elimination/address_unallocated.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/address_unallocated.move_elimination.stderr new file mode 100644 index 0000000000000..5cc1a111ea807 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/address_unallocated.move_elimination.stderr @@ -0,0 +1,13 @@ +error: Undefined Behavior: accessing a live but unallocated local variable + --> tests/fail/move_elimination/address_unallocated.rs:LL:CC + | +LL | ptr = &raw mut value; + | ^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/address_unallocated.rs b/src/tools/miri/tests/fail/move_elimination/address_unallocated.rs new file mode 100644 index 0000000000000..202f1e4927102 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/address_unallocated.rs @@ -0,0 +1,20 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let ptr: *mut bool; + let value: bool; + { + ptr = &raw mut value; //~[move_elimination] ERROR: live but unallocated + *ptr = true; + Return() + } + } +} diff --git a/src/tools/miri/tests/fail/move_elimination/use_after_move.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/use_after_move.move_elimination.stderr new file mode 100644 index 0000000000000..56fc23f1c0aa3 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/use_after_move.move_elimination.stderr @@ -0,0 +1,29 @@ +error: Undefined Behavior: memory access failed: ALLOC has been freed, so this pointer is dangling + --> tests/fail/move_elimination/use_after_move.rs:LL:CC + | +LL | read = (*ptr).0; + | ^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information +help: ALLOC was allocated here: + --> tests/fail/move_elimination/use_after_move.rs:LL:CC + | +LL | / mir! { +LL | | let value: (u8, u8); +LL | | let moved: (u8, u8); +LL | | let ptr: *const (u8, u8); +... | +LL | | } + | |_____^ +help: ALLOC was deallocated here: + --> tests/fail/move_elimination/use_after_move.rs:LL:CC + | +LL | moved = Move(value); + | ^^^^^^^^^^^^^^^^^^^ + = note: this error originates in the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/use_after_move.rs b/src/tools/miri/tests/fail/move_elimination/use_after_move.rs new file mode 100644 index 0000000000000..ab312f47046a8 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/use_after_move.rs @@ -0,0 +1,24 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: (u8, u8); + let moved: (u8, u8); + let ptr: *const (u8, u8); + let read: u8; + { + value = (1, 2); + ptr = &raw const value; + moved = Move(value); + read = (*ptr).0; //~[move_elimination] ERROR: has been freed + Return() + } + } +} diff --git a/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.move_elimination.stderr b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.move_elimination.stderr new file mode 100644 index 0000000000000..cb81052db28e2 --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.move_elimination.stderr @@ -0,0 +1,29 @@ +error: Undefined Behavior: memory access failed: ALLOC has been freed, so this pointer is dangling + --> tests/fail/move_elimination/use_after_move_in_statement.rs:LL:CC + | +LL | result = (Move(value), (*ptr).0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here + | + = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior + = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information +help: ALLOC was allocated here: + --> tests/fail/move_elimination/use_after_move_in_statement.rs:LL:CC + | +LL | / mir! { +LL | | let value: (u8, u8); +LL | | let ptr: *const (u8, u8); +LL | | let result: ((u8, u8), u8); +... | +LL | | } + | |_____^ +help: ALLOC was deallocated here: + --> tests/fail/move_elimination/use_after_move_in_statement.rs:LL:CC + | +LL | result = (Move(value), (*ptr).0); + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + = note: this error originates in the macro `mir` (in Nightly builds, run with -Z macro-backtrace for more info) + +note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace + +error: aborting due to 1 previous error + diff --git a/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.rs b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.rs new file mode 100644 index 0000000000000..db103d79dcf8e --- /dev/null +++ b/src/tools/miri/tests/fail/move_elimination/use_after_move_in_statement.rs @@ -0,0 +1,22 @@ +//@revisions: normal move_elimination +//@[normal]check-pass +//@[move_elimination]compile-flags: -Zmir-move-elimination + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn main() { + mir! { + let value: (u8, u8); + let ptr: *const (u8, u8); + let result: ((u8, u8), u8); + { + value = (1, 2); + ptr = &raw const value; + result = (Move(value), (*ptr).0); //~[move_elimination] ERROR: has been freed + Return() + } + } +} diff --git a/src/tools/miri/tests/pass/move_elimination_zsts.rs b/src/tools/miri/tests/pass/move_elimination_zsts.rs new file mode 100644 index 0000000000000..d4f552e655ed5 --- /dev/null +++ b/src/tools/miri/tests/pass/move_elimination_zsts.rs @@ -0,0 +1,50 @@ +//@compile-flags: -Zmir-move-elimination -Zmir-opt-level=0 + +#![feature(core_intrinsics, custom_mir)] + +use std::intrinsics::mir::*; + +struct Zst; + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn uninitialized_unit() { + mir! { + { + Return() + } + } +} + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn uninitialized_zst() -> Zst { + mir! { + { + Return() + } + } +} + +#[custom_mir(dialect = "runtime", phase = "optimized")] +fn move_keeps_zst_address() -> (*const Zst, *const Zst) { + mir! { + let value: Zst; + let moved: Zst; + let before: *const Zst; + let after: *const Zst; + { + value = Zst; + before = &raw const value; + moved = Move(value); + after = &raw const value; + RET = (before, after); + Return() + } + } +} + +fn main() { + uninitialized_unit(); + let _ = uninitialized_zst(); + let (before, after) = move_keeps_zst_address(); + assert_eq!(before, after); +} diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff new file mode 100644 index 0000000000000..64d0fe11bfa9f --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff @@ -0,0 +1,29 @@ +- // MIR for `aggregate_indirect_source_alias` before MoveElimination ++ // MIR for `aggregate_indirect_source_alias` after MoveElimination + + fn aggregate_indirect_source_alias() -> (u8, u8) { + let mut _0: (u8, u8); + let mut _1: u8; + let mut _2: *const u8; + let mut _3: (u8, u8); ++ let mut _4: u8; + + bb0: { +- _1 = const 1_u8; +- _2 = &raw const _1; +- _3 = (copy (*_2), move _1); +- _0 = copy _3; ++ (_0.1: u8) = const 1_u8; ++ StorageLive(_2); ++ _2 = &raw const (_0.1: u8); ++ StorageLive(_4); ++ _4 = no_retag copy (*_2); ++ (_0.0: u8) = no_retag move _4; ++ nop; ++ StorageDead(_4); ++ StorageDead(_2); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff new file mode 100644 index 0000000000000..481d3ef60feb8 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff @@ -0,0 +1,84 @@ +- // MIR for `aggregate_swap` before MoveElimination ++ // MIR for `aggregate_swap` after MoveElimination + + fn aggregate_swap(_1: u8, _2: u8) -> (u8, u8) { + debug x => _1; + debug y => _2; + let mut _0: (u8, u8); + let mut _3: (u8, u8); + let mut _4: u8; + let mut _5: u8; + let mut _8: u8; + let mut _9: u8; ++ let mut _10: u8; + scope 1 { +- debug pair => _3; ++ debug pair => _0; + let _6: u8; + scope 2 { +- debug a => _6; ++ debug a => _9; + let _7: u8; + scope 3 { +- debug b => _7; ++ debug b => (_0.1: u8); + } + } + } + + bb0: { +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _1; +- StorageLive(_5); +- _5 = copy _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- StorageLive(_6); +- _6 = copy (_3.0: u8); +- StorageLive(_7); +- _7 = copy (_3.1: u8); +- StorageLive(_8); +- _8 = copy _7; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = (move _1, move _2); ++ nop; ++ nop; ++ nop; + StorageLive(_9); +- _9 = copy _6; +- _3 = (move _8, move _9); ++ _9 = copy (_0.0: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_10); ++ _10 = no_retag move (_0.1: u8); ++ (_0.0: u8) = no_retag move _10; ++ (_0.1: u8) = no_retag move _9; ++ nop; ++ StorageDead(_10); + StorageDead(_9); +- StorageDead(_8); +- _0 = copy _3; +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff new file mode 100644 index 0000000000000..27f6c10549040 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff @@ -0,0 +1,90 @@ +- // MIR for `mixed_aggregate_aliasing` before MoveElimination ++ // MIR for `mixed_aggregate_aliasing` after MoveElimination + + fn mixed_aggregate_aliasing(_1: bool, _2: u8) -> Triple { + debug flag => _1; + debug z => _2; + let mut _0: Triple; + let _3: Triple; + let mut _5: bool; + let mut _6: u8; + let mut _7: u8; + let mut _8: u8; ++ let mut _9: u8; + scope 1 { +- debug input => _3; ++ debug input => _0; + let _4: Triple; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_3); +- _3 = opaque_triple() -> [return: bb1, unwind unreachable]; ++ nop; ++ _0 = opaque_triple() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _1; +- switchInt(move _5) -> [0: bb3, otherwise: bb2]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb2]; + } + + bb2: { +- _4 = copy _3; ++ nop; + goto -> bb4; + } + + bb3: { ++ nop; + StorageLive(_6); +- _6 = copy (_3.1: u8); +- StorageLive(_7); +- _7 = copy (_3.0: u8); +- StorageLive(_8); +- _8 = copy _2; +- _4 = Triple(move _6, move _7, move _8); +- StorageDead(_8); +- StorageDead(_7); ++ _6 = copy (_0.1: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_9); ++ _9 = no_retag move (_0.0: u8); ++ (_0.0: u8) = no_retag move _6; ++ (_0.1: u8) = no_retag move _9; ++ (_0.2: u8) = no_retag move _2; ++ nop; ++ StorageDead(_9); + StorageDead(_6); ++ nop; ++ nop; ++ nop; + goto -> bb4; + } + + bb4: { +- StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.rs b/tests/mir-opt/move-elimination/alias_fixup.rs new file mode 100644 index 0000000000000..29d20a61488fa --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.rs @@ -0,0 +1,95 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir)] +#![allow(dead_code)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +#[derive(Copy, Clone)] +pub struct Triple(u8, u8, u8); + +pub union U { + a: [u8; 4], + b: [u8; 4], +} + +unsafe extern "C" { + safe fn opaque_triple() -> Triple; +} + +// EMIT_MIR alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff +pub fn mixed_aggregate_aliasing(flag: bool, z: u8) -> Triple { + // This checks an aggregate assignment on one branch after the other branch + // remaps the input into the return place: overlapping field reads are + // hoisted through temporaries before writing back into the return place. + // CHECK-LABEL: fn mixed_aggregate_aliasing( + // CHECK: debug z => _2; + // CHECK: debug input => _0; + // CHECK: debug out => _0; + // CHECK: [[field1:_.*]] = copy (_0.1: u8); + // CHECK: [[field0:_.*]] = no_retag move (_0.0: u8); + // CHECK: (_0.0: u8) = no_retag move [[field1]]; + // CHECK: (_0.1: u8) = no_retag move [[field0]]; + // CHECK: (_0.2: u8) = no_retag move _2; + let input = opaque_triple(); + let out = if flag { input } else { Triple(input.1, input.0, z) }; + out +} + +// EMIT_MIR alias_fixup.aggregate_swap.MoveElimination.diff +pub fn aggregate_swap(x: u8, y: u8) -> (u8, u8) { + // This checks that an aggregate swap-like assignment is safe after any + // remapping that makes source fields share storage with destination fields. + // CHECK-LABEL: fn aggregate_swap( + // CHECK: debug pair => _0; + // CHECK: [[saved:_.*]] = copy (_0.0: u8); + // CHECK: [[tmp:_.*]] = no_retag move (_0.1: u8); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + // CHECK: (_0.1: u8) = no_retag move [[saved]]; + let mut pair = (x, y); + let a = pair.0; + let b = pair.1; + pair = (b, a); + pair +} + +// EMIT_MIR alias_fixup.simple_partial_alias.MoveElimination.diff +pub fn simple_partial_alias(x: [u8; 4]) -> U { + // This checks a non-aggregate assignment involving two same-typed union + // fields, which are conservatively treated as aliasing. + // CHECK-LABEL: fn simple_partial_alias( + // CHECK: debug u => _0; + // CHECK: _0 = U { a: move _1 }; + // CHECK: [[tmp:_.*]] = copy (_0.0: [u8; 4]); + // CHECK: (_0.1: [u8; 4]) = move [[tmp]]; + let mut u = U { a: x }; + let tmp = unsafe { u.a }; + u.b = tmp; + u +} + +// EMIT_MIR alias_fixup.aggregate_indirect_source_alias.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_indirect_source_alias() -> (u8, u8) { + // This checks that, when an aggregate has a direct operand aliasing the + // destination, indirect operands are also hoisted before field writes. + // CHECK-LABEL: fn aggregate_indirect_source_alias( + // CHECK: [[p:_.*]] = &raw const (_0.1: u8); + // CHECK: [[tmp:_.*]] = no_retag copy (*[[p]]); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + mir! { + let a: u8; + let p: *const u8; + let out: (u8, u8); + + { + a = 1_u8; + p = &raw const a; + out = (*p, Move(a)); + RET = out; + Return() + } + } +} diff --git a/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff new file mode 100644 index 0000000000000..af47eb7347d2a --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff @@ -0,0 +1,52 @@ +- // MIR for `simple_partial_alias` before MoveElimination ++ // MIR for `simple_partial_alias` after MoveElimination + + fn simple_partial_alias(_1: [u8; 4]) -> U { + debug x => _1; + let mut _0: U; + let mut _2: U; + let mut _3: [u8; 4]; + let mut _5: [u8; 4]; + scope 1 { +- debug u => _2; ++ debug u => _0; + let _4: [u8; 4]; + scope 2 { +- debug tmp => _4; ++ debug tmp => _5; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = U { a: move _3 }; +- StorageDead(_3); +- StorageLive(_4); +- _4 = copy (_2.0: [u8; 4]); ++ nop; ++ nop; ++ nop; ++ _0 = U { a: move _1 }; ++ nop; ++ nop; + StorageLive(_5); +- _5 = copy _4; +- (_2.1: [u8; 4]) = move _5; ++ _5 = copy (_0.0: [u8; 4]); ++ nop; ++ nop; ++ (_0.1: [u8; 4]) = move _5; + StorageDead(_5); +- _0 = move _2; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..01938d10da6c5 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff @@ -0,0 +1,67 @@ +- // MIR for `array_aggregate` before MoveElimination ++ // MIR for `array_aggregate` after MoveElimination + + fn array_aggregate() -> [[u8; 8]; 3] { + let mut _0: [[u8; 8]; 3]; + let _1: [u8; 8]; + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + let mut _6: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _0[0 of 1]; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _0[1 of 2]; + let _3: [u8; 8]; + scope 3 { +- debug c => _3; ++ debug c => _0[2 of 3]; + } + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = [const 3_u8; 8]; +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- StorageLive(_6); +- _6 = move _3; +- _0 = [move _4, move _5, move _6]; +- StorageDead(_6); +- StorageDead(_5); +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ _0[0 of 1] = [const 1_u8; 8]; ++ nop; ++ _0[1 of 2] = [const 2_u8; 8]; ++ nop; ++ _0[2 of 3] = [const 3_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..5f40d8a3c82fd --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `enum_aggregate` before MoveElimination ++ // MIR for `enum_aggregate` after MoveElimination + + fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + let mut _0: std::result::Result<([u8; 8], [u8; 8]), ()>; + let _1: [u8; 8]; + let mut _3: ([u8; 8], [u8; 8]); + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- StorageLive(_4); +- _4 = move _1; +- StorageLive(_5); +- _5 = move _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- _0 = Result::<([u8; 8], [u8; 8]), ()>::Ok(move _3); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ discriminant(_0) = 0; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..8a07af2edceaf --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff @@ -0,0 +1,50 @@ +- // MIR for `nrvo_borrowed` before MoveElimination ++ // MIR for `nrvo_borrowed` after MoveElimination + + fn nrvo_borrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let mut _1: [u8; 8]; + let _2: (); + let mut _3: &mut [u8; 8]; + let mut _4: &mut [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- StorageLive(_3); ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; ++ nop; + StorageLive(_4); +- _4 = &mut _1; ++ _4 = &mut _0; ++ StorageLive(_3); + _3 = &mut (*_4); ++ StorageDead(_4); ++ StorageLive(_2); + _2 = init(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageDead(_4); + StorageDead(_2); +- _0 = move _1; +- StorageDead(_1); ++ StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff new file mode 100644 index 0000000000000..7a44d4b598211 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff @@ -0,0 +1,24 @@ +- // MIR for `nrvo_unborrowed` before MoveElimination ++ // MIR for `nrvo_unborrowed` after MoveElimination + + fn nrvo_unborrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let _1: [u8; 8]; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- _0 = move _1; +- StorageDead(_1); ++ nop; ++ _0 = [const 1_u8; 8]; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.rs b/tests/mir-opt/move-elimination/basic.rs new file mode 100644 index 0000000000000..96f1776bebef1 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.rs @@ -0,0 +1,80 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+TailCopyToMove + +struct Pair { + a: [u8; 8], + b: [u8; 8], +} + +fn init(_: &mut [u8; 8]) {} + +// EMIT_MIR basic.nrvo_unborrowed.MoveElimination.diff +pub fn nrvo_unborrowed() -> [u8; 8] { + // This checks the simplest NRVO-style case: the local should be merged with + // the return place even though it has `Copy` type. + // CHECK-LABEL: fn nrvo_unborrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + let buf = [1; 8]; + buf +} + +// EMIT_MIR basic.nrvo_borrowed.MoveElimination.diff +pub fn nrvo_borrowed() -> [u8; 8] { + // This checks that taking a temporary mutable borrow does not prevent + // merging a `Copy` local once the borrow has ended. + // CHECK-LABEL: fn nrvo_borrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = [const 1_u8; 8] + // CHECK: init(move {{_.*}}) + // CHECK-NOT: _0 = move + let mut buf = [1; 8]; + init(&mut buf); + buf +} + +// EMIT_MIR basic.struct_aggregate.MoveElimination.diff +pub fn struct_aggregate() -> Pair { + // This checks aggregate field remapping: the field locals can live directly + // in the return place's fields. + // CHECK-LABEL: fn struct_aggregate( + // CHECK: debug a => (_0.0: [u8; 8]); + // CHECK: debug b => (_0.1: [u8; 8]); + // CHECK: (_0.0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (_0.1: [u8; 8]) = [const 2_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + Pair { a, b } +} + +// EMIT_MIR basic.enum_aggregate.MoveElimination.diff +pub fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + // This checks aggregate field remapping for enums: the payload fields can + // be written directly and then the discriminant is set for the variant. + // CHECK-LABEL: fn enum_aggregate( + // CHECK: debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + // CHECK: debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; + // CHECK: discriminant(_0) = 0; + let a = [1; 8]; + let b = [2; 8]; + Result::Ok((a, b)) +} + +// EMIT_MIR basic.array_aggregate.MoveElimination.diff +pub fn array_aggregate() -> [[u8; 8]; 3] { + // This checks aggregate remapping for arrays, which uses ConstantIndex + // projections rather than field projections. + // CHECK-LABEL: fn array_aggregate( + // CHECK: debug a => _0[0 of 1]; + // CHECK: debug b => _0[1 of 2]; + // CHECK: debug c => _0[2 of 3]; + // CHECK: _0[0 of 1] = [const 1_u8; 8]; + // CHECK: _0[1 of 2] = [const 2_u8; 8]; + // CHECK: _0[2 of 3] = [const 3_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + let c = [3; 8]; + [a, b, c] +} diff --git a/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..b03c4d2f3cff9 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `struct_aggregate` before MoveElimination ++ // MIR for `struct_aggregate` after MoveElimination + + fn struct_aggregate() -> Pair { + let mut _0: Pair; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (_0.0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (_0.1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = move _1; +- StorageLive(_4); +- _4 = move _2; +- _0 = Pair { a: move _3, b: move _4 }; +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (_0.0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (_0.1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff new file mode 100644 index 0000000000000..895dab2802597 --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.dse_guard.MoveElimination.diff @@ -0,0 +1,97 @@ +- // MIR for `dse_guard` before MoveElimination ++ // MIR for `dse_guard` after MoveElimination + + fn dse_guard() -> () { + let mut _0: (); + let mut _1: Fields; + let mut _3: Fields; + let mut _4: Fields; + let _5: (); + let mut _6: *const Fields; + let mut _7: Fields; + let _8: (); + let mut _9: *const Fields; + scope 1 { +- debug a => _1; ++ debug a => _7; + let mut _2: Fields; + scope 2 { + debug b => _2; + } + } + + bb0: { +- StorageLive(_1); ++ nop; ++ nop; ++ nop; + StorageLive(_2); +- StorageLive(_3); +- _3 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _2 = move _3; +- StorageDead(_3); +- StorageLive(_4); +- _4 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ StorageLive(_7); ++ _7 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- _1 = move _4; +- StorageDead(_4); +- StorageLive(_5); ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_6); +- _6 = &raw const _1; ++ _6 = &raw const _7; ++ StorageLive(_5); + _5 = observe(move _6) -> [return: bb3, unwind unreachable]; + } + + bb3: { +- StorageDead(_6); + StorageDead(_5); +- StorageLive(_7); +- _7 = move _1; ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; + _2 = move _7; + StorageDead(_7); +- StorageLive(_8); ++ nop; ++ nop; ++ nop; + StorageLive(_9); + _9 = &raw const _2; ++ StorageLive(_8); + _8 = observe(move _9) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- StorageDead(_9); + StorageDead(_8); ++ StorageDead(_9); ++ nop; ++ nop; + _0 = const (); ++ nop; + StorageDead(_2); +- StorageDead(_1); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/dse.rs b/tests/mir-opt/move-elimination/dse.rs new file mode 100644 index 0000000000000..268b54fa96ae6 --- /dev/null +++ b/tests/mir-opt/move-elimination/dse.rs @@ -0,0 +1,38 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+DeadStoreElimination-initial + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR dse.dse_guard.MoveElimination.diff +pub fn dse_guard() { + // This guards the RFC soundness hazard: DSE must not remove the first write + // to `b`, because that write keeps `b`'s address-observed lifetime + // overlapping with `a` and prevents the later move from being eliminated. + // CHECK-LABEL: fn dse_guard( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: StorageLive([[b]]); + // CHECK: [[b]] = make_fields(const 0_u8) + // CHECK: StorageLive([[a]]); + // CHECK: [[a]] = make_fields(const 1_u8) + // CHECK: observe(move + // CHECK: [[b]] = move [[a]] + // CHECK: observe(move + let mut a; + let mut b; + + b = make_fields(0); + + a = make_fields(1); + observe(&raw const a); + b = a; + observe(&raw const b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..6aa7c42a2ba08 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff @@ -0,0 +1,20 @@ +- // MIR for `index_local_not_projected` before MoveElimination ++ // MIR for `index_local_not_projected` after MoveElimination + + fn index_local_not_projected(_1: [usize; 4]) -> [usize; 1] { + let mut _0: [usize; 1]; + let mut _2: usize; + let mut _3: usize; + + bb0: { ++ StorageLive(_2); + _2 = const 2_usize; ++ StorageLive(_3); + _3 = copy _1[_2]; ++ StorageDead(_3); + _0 = [copy _2]; ++ StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff new file mode 100644 index 0000000000000..51db7a9e85374 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff @@ -0,0 +1,136 @@ +- // MIR for `overlapping_lifetimes` before MoveElimination ++ // MIR for `overlapping_lifetimes` after MoveElimination + + fn overlapping_lifetimes(_1: bool) -> Fields { + debug flag => _1; + let mut _0: Fields; + let _2: Fields; + let _4: (); + let mut _5: *const Fields; + let _6: (); + let mut _7: bool; + let mut _8: Fields; + let mut _9: Fields; + let mut _10: Fields; + let _11: (); + let mut _12: *const Fields; + scope 1 { +- debug src => _2; ++ debug src => _10; + let mut _3: Fields; + scope 2 { +- debug dst => _3; ++ debug dst => _0; + } + } + + bb0: { +- StorageLive(_2); +- _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ nop; ++ StorageLive(_10); ++ _10 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_3); +- StorageLive(_4); ++ nop; ++ nop; ++ nop; + StorageLive(_5); +- _5 = &raw const _2; ++ _5 = &raw const _10; ++ StorageLive(_4); + _4 = observe(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); +- StorageLive(_6); +- StorageLive(_7); +- _7 = copy _1; +- switchInt(move _7) -> [0: bb5, otherwise: bb3]; ++ StorageDead(_5); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb5, otherwise: bb3]; + } + + bb3: { +- StorageLive(_8); +- _8 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; ++ nop; ++ _0 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- _3 = move _8; +- StorageDead(_8); +- StorageLive(_9); +- _9 = move _2; +- _3 = move _9; +- StorageDead(_9); ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = move _10; ++ StorageDead(_10); ++ nop; ++ StorageLive(_6); + _6 = const (); ++ StorageDead(_6); + goto -> bb6; + } + + bb5: { +- StorageLive(_10); +- _10 = move _2; +- _3 = move _10; ++ nop; ++ nop; ++ _0 = move _10; + StorageDead(_10); ++ nop; ++ StorageLive(_6); + _6 = const (); ++ StorageDead(_6); + goto -> bb6; + } + + bb6: { +- StorageDead(_7); +- StorageDead(_6); +- StorageLive(_11); ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_12); +- _12 = &raw const _3; ++ _12 = &raw const _0; ++ StorageLive(_11); + _11 = observe(move _12) -> [return: bb7, unwind unreachable]; + } + + bb7: { +- StorageDead(_12); + StorageDead(_11); +- _0 = move _3; +- StorageDead(_3); +- StorageDead(_2); ++ StorageDead(_12); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..af6d2b34a0439 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `packed_fields_not_projected` before MoveElimination ++ // MIR for `packed_fields_not_projected` after MoveElimination + + fn packed_fields_not_projected() -> Packed { + let mut _0: Packed; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _3; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _4; + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; ++ nop; + StorageLive(_3); +- _3 = copy _1; ++ _3 = [const 1_u8; 8]; ++ nop; + StorageLive(_4); +- _4 = copy _2; ++ _4 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = Packed { a: move _3, b: move _4 }; +- StorageDead(_4); + StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ StorageDead(_4); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.rs b/tests/mir-opt/move-elimination/exclusions.rs new file mode 100644 index 0000000000000..2e45946d87ef1 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rs @@ -0,0 +1,118 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(core_intrinsics, custom_mir, repr_simd)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +#[repr(packed)] +struct Packed { + a: [u8; 8], + b: [u8; 8], +} + +#[repr(simd)] +struct U32x4([u32; 4]); + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR exclusions.index_local_not_projected.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_local_not_projected(a: [usize; 4]) -> [usize; 1] { + // This checks that a local used as an array index is kept as a bare local, + // because it cannot later be rewritten to a projection like `_0[0]`. + // CHECK-LABEL: fn index_local_not_projected( + // CHECK: [[idx:_.*]] = const 2_usize; + // CHECK: {{.*}} = copy _1[{{.*}}[[idx]]{{.*}}]; + // CHECK: _0 = [copy [[idx]]]; + mir! { + let idx: usize; + let b: usize; + + { + idx = 2usize; + b = a[idx]; + RET = [idx]; + Return() + } + } +} + +// EMIT_MIR exclusions.packed_fields_not_projected.MoveElimination.diff +pub fn packed_fields_not_projected() -> Packed { + // This checks that aggregate fields are not remapped into packed struct + // fields, which could create unaligned projected places. + // CHECK-LABEL: fn packed_fields_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: _0 = Packed { a: move [[a]], b: move [[b]] }; + let a = [1; 8]; + let b = [2; 8]; + Packed { a, b } +} + +// EMIT_MIR exclusions.simd_field_not_projected.MoveElimination.diff +pub fn simd_field_not_projected() -> U32x4 { + // This checks that aggregate fields are not remapped into repr(simd) ADTs, + // since optimized MIR must not project into SIMD vectors. + // CHECK-LABEL: fn simd_field_not_projected( + // CHECK: debug lanes => [[lanes:_.*]]; + // CHECK: _0 = U32x4(move [[lanes]]); + let lanes = [1, 2, 3, 4]; + U32x4(lanes) +} + +// EMIT_MIR exclusions.overlapping_lifetimes.MoveElimination.diff +pub fn overlapping_lifetimes(flag: bool) -> Fields { + // This checks the liveness-matrix overlap test for an address-observed + // move-only local: `src` and `dst` only overlap on one branch, but that is + // enough to reject merging them for the whole function. + // CHECK-LABEL: fn overlapping_lifetimes( + // CHECK: debug flag => _1; + // CHECK: debug src => [[src:_[1-9][0-9]*]]; + // CHECK: debug dst => _0; + // CHECK: &raw const [[src]]; + // CHECK: observe + // CHECK: switchInt(move _1) + // CHECK: _0 = make_fields(const 1_u8) + // CHECK: _0 = move [[src]]; + // CHECK: &raw const _0; + // CHECK: observe + let src = make_fields(0); + let mut dst; + observe(&raw const src); + if flag { + dst = make_fields(1); + dst = src; + } else { + dst = src; + } + observe(&raw const dst); + dst +} + +// EMIT_MIR exclusions.rust_call_tuple_not_projected.MoveElimination.diff +pub fn rust_call_tuple_not_projected(f: F) { + // This checks that locals are not remapped into the tuple argument passed + // to a rust-call ABI function. If the tuple itself is never borrowed, alias + // analysis can trivially see that accesses to one argument don't affect the + // others. Merging the arguments into tuple fields from the start can hide + // that independence. + // CHECK-LABEL: fn rust_call_tuple_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[tuple:_.*]] = (move [[a]], move [[b]]); + // CHECK: >::call_once(move _1, move [[tuple]]) + let a = [1; 8]; + let b = [2; 8]; + f(a, b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..fcaf35ddeb8b0 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff @@ -0,0 +1,77 @@ +- // MIR for `rust_call_tuple_not_projected` before MoveElimination ++ // MIR for `rust_call_tuple_not_projected` after MoveElimination + + fn rust_call_tuple_not_projected(_1: F) -> () { + debug f => _1; + let mut _0: (); + let _2: [u8; 8]; + let _4: (); + let mut _5: F; + let mut _6: ([u8; 8], [u8; 8]); + let mut _7: [u8; 8]; + let mut _8: [u8; 8]; + scope 1 { +- debug a => _2; ++ debug a => _7; + let _3: [u8; 8]; + scope 2 { +- debug b => _3; ++ debug b => _8; + } + } + + bb0: { +- StorageLive(_2); +- _2 = [const 1_u8; 8]; +- StorageLive(_3); +- _3 = [const 2_u8; 8]; +- StorageLive(_4); +- StorageLive(_5); +- _5 = move _1; +- StorageLive(_6); ++ nop; + StorageLive(_7); +- _7 = copy _2; ++ _7 = [const 1_u8; 8]; ++ nop; + StorageLive(_8); +- _8 = copy _3; ++ _8 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_6); + _6 = (move _7, move _8); +- _4 = >::call_once(move _5, move _6) -> [return: bb1, unwind unreachable]; ++ StorageDead(_7); ++ StorageDead(_8); ++ StorageLive(_4); ++ _4 = >::call_once(move _1, move _6) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_8); +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = const (); +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..4844207200f34 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff @@ -0,0 +1,30 @@ +- // MIR for `simd_field_not_projected` before MoveElimination ++ // MIR for `simd_field_not_projected` after MoveElimination + + fn simd_field_not_projected() -> U32x4 { + let mut _0: U32x4; + let _1: [u32; 4]; + let mut _2: [u32; 4]; + scope 1 { +- debug lanes => _1; ++ debug lanes => _2; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; + StorageLive(_2); +- _2 = copy _1; ++ _2 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; ++ nop; + _0 = U32x4(move _2); + StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff new file mode 100644 index 0000000000000..dfc680dc01ff2 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff @@ -0,0 +1,61 @@ +- // MIR for `address_observed_storage_dead_at_end` before MoveElimination ++ // MIR for `address_observed_storage_dead_at_end` after MoveElimination + + fn address_observed_storage_dead_at_end(_1: bool) -> () { + debug flag => _1; + let mut _0: (); + let _2: u32; + let mut _3: bool; + let _4: *const u32; + let mut _5: *const u32; + scope 1 { + debug x => _2; + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- switchInt(move _3) -> [0: bb3, otherwise: bb1]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb1]; + } + + bb1: { ++ StorageLive(_2); + _2 = const 1_u32; +- StorageLive(_4); ++ nop; ++ nop; + StorageLive(_5); + _5 = &raw const _2; ++ StorageLive(_4); + _4 = opaque::<*const u32>(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_5); ++ nop; ++ nop; + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); + goto -> bb4; + } + + bb4: { +- StorageDead(_3); ++ nop; ++ nop; + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff new file mode 100644 index 0000000000000..c28ca81eb5b9c --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff @@ -0,0 +1,87 @@ +- // MIR for `borrowed_not_shortened_to_last_direct_use` before MoveElimination ++ // MIR for `borrowed_not_shortened_to_last_direct_use` after MoveElimination + + fn borrowed_not_shortened_to_last_direct_use(_1: u32) -> () { + debug x => _1; + let mut _0: (); + let _2: u32; + let mut _3: u32; + let _6: u32; + let mut _7: u32; + let mut _8: u32; + let mut _9: u32; + scope 1 { + debug a => _2; + let _4: &u32; + scope 2 { + debug r => _4; + let _5: u32; + scope 3 { +- debug out => _5; ++ debug out => _9; + } + } + } + + bb0: { ++ nop; ++ nop; ++ nop; + StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ _2 = opaque::(move _1) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); ++ nop; ++ nop; + StorageLive(_4); + _4 = &_2; +- StorageLive(_5); +- _5 = copy _2; +- StorageLive(_6); +- StorageLive(_7); ++ nop; ++ StorageLive(_9); ++ _9 = copy _2; ++ nop; ++ nop; ++ nop; + StorageLive(_8); + _8 = copy (*_4); +- StorageLive(_9); +- _9 = copy _5; ++ StorageDead(_4); ++ nop; ++ nop; ++ StorageLive(_7); + _7 = Add(move _8, move _9); +- StorageDead(_9); + StorageDead(_8); ++ StorageDead(_9); ++ nop; ++ nop; ++ StorageLive(_6); + _6 = opaque::(move _7) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_7); + StorageDead(_6); ++ StorageDead(_7); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_5); +- StorageDead(_4); ++ nop; ++ nop; ++ nop; + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff new file mode 100644 index 0000000000000..20a5abf62e439 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff @@ -0,0 +1,35 @@ +- // MIR for `critical_edge_split_for_storage_live` before MoveElimination ++ // MIR for `critical_edge_split_for_storage_live` after MoveElimination + + fn critical_edge_split_for_storage_live(_1: bool) -> () { + debug x => _2; + let mut _0: (); + let mut _2: u32; + let mut _3: *const u32; + + bb0: { +- switchInt(copy _1) -> [1: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [1: bb1, otherwise: bb3]; + } + + bb1: { ++ StorageLive(_2); + _2 = const 1_u32; ++ StorageLive(_3); + _3 = &raw const _2; + _3 = opaque::<*const u32>(copy _3) -> [return: bb2, unwind unreachable]; + } + + bb2: { ++ StorageDead(_3); + _2 = const 2_u32; ++ StorageDead(_2); + return; ++ } ++ ++ bb3: { ++ StorageLive(_2); ++ goto -> bb2; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.rs b/tests/mir-opt/move-elimination/storage.rs new file mode 100644 index 0000000000000..c699e4591c22c --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.rs @@ -0,0 +1,236 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zlint-mir=false + +#![feature(custom_mir, core_intrinsics)] + +use std::intrinsics::mir::*; + +fn opaque(x: T) -> T { + x +} + +// EMIT_MIR storage.shorten_non_borrowed.MoveElimination.diff +pub fn shorten_non_borrowed(x: u32) { + // This checks that reconstruction can shorten a non-borrowed local's + // storage to its last use instead of keeping lexical storage markers. + // CHECK-LABEL: fn shorten_non_borrowed( + // CHECK: debug a => [[short:_.*]]; + // CHECK: debug b => [[short]]; + // CHECK: StorageLive([[short]]); + // CHECK: [[short]] = opaque::( + // CHECK: opaque::(move [[short]]) -> [return: [[short_ret:bb.*]], + // CHECK: [[short_ret]]: { + // CHECK: StorageDead([[short]]); + // CHECK: opaque::( + let a = opaque(x); + let b = a; + opaque(b); + opaque(x); +} + +// EMIT_MIR storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff +pub fn borrowed_not_shortened_to_last_direct_use(x: u32) { + // This checks that a borrowed local is not shortened merely to its last + // direct use; the borrow keeps its storage live while the reference exists. + // CHECK-LABEL: fn borrowed_not_shortened_to_last_direct_use( + // CHECK: debug a => [[borrowed:_.*]]; + // CHECK: debug r => [[borrow_ref:_.*]]; + // CHECK: debug out => [[out:_.*]]; + // CHECK: StorageLive([[borrowed]]); + // CHECK: [[borrowed]] = opaque::(move _1) + // CHECK: [[borrow_ref]] = &[[borrowed]]; + // CHECK: [[out]] = copy [[borrowed]]; + // CHECK: copy (*[[borrow_ref]]); + // CHECK: StorageDead([[borrowed]]); + let a = opaque(x); + let r = &a; + let out = a; + opaque(*r + out); +} + +// EMIT_MIR storage.storage_live_moved_to_branch.MoveElimination.diff +pub fn storage_live_moved_to_branch(flag: bool) { + // This checks storage reconstruction can shrink a local declared before a + // branch so its storage is live only on the arm where it is initialized. + // CHECK-LABEL: fn storage_live_moved_to_branch( + // CHECK: debug x => [[branch_tmp:_.*]]; + // CHECK: switchInt(move _1) -> [0: bb3, otherwise: bb1]; + // CHECK: bb1: { + // CHECK: StorageLive([[branch_tmp]]); + // CHECK: [[branch_tmp]] = const 1_u32; + // CHECK: opaque::(move [[branch_tmp]]) -> [return: [[branch_ret:bb.*]], + // CHECK: [[branch_ret]]: { + // CHECK: StorageDead([[branch_tmp]]); + let x: u32; + if flag { + x = 1; + opaque(x); + } +} + +// EMIT_MIR storage.address_observed_storage_dead_at_end.MoveElimination.diff +pub fn address_observed_storage_dead_at_end(flag: bool) { + // This checks that an address-observed local declared before a branch still + // has StorageLive moved into the initialized arm without adding one to the + // uninitialized arm, but StorageDead remains at the end of the function + // instead of being shortened to the last direct use. + // CHECK-LABEL: fn address_observed_storage_dead_at_end( + // CHECK: debug x => [[addr_tmp:_.*]]; + // CHECK: switchInt(move _1) -> [0: [[skip:bb.*]], otherwise: [[init:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[addr_tmp]]); + // CHECK: [[addr_tmp]] = const 1_u32; + // CHECK: &raw const [[addr_tmp]]; + // CHECK: opaque::<*const u32> + // CHECK-NOT: StorageDead([[addr_tmp]]); + // CHECK: [[skip]]: { + // CHECK-NOT: StorageLive([[addr_tmp]]); + // CHECK: {{bb.*}}: { + // CHECK: StorageDead([[addr_tmp]]); + let x: u32; + if flag { + x = 1; + opaque(&raw const x); + } +} + +// EMIT_MIR storage.terminator_end_storage_dead_in_successor.MoveElimination.diff +pub fn terminator_end_storage_dead_in_successor(x: u32) -> u32 { + // This checks storage reconstruction when the last use of a local is as a + // call argument in a terminator. + // CHECK-LABEL: fn terminator_end_storage_dead_in_successor( + // CHECK: debug tmp => [[term_tmp:_.*]]; + // CHECK: StorageLive([[term_tmp]]); + // CHECK: [[term_tmp]] = opaque::(move _1) + // CHECK: opaque::(move [[term_tmp]]) -> [return: [[term_ret:bb.*]], + // CHECK: [[term_ret]]: { + // CHECK-NEXT: StorageDead([[term_tmp]]); + let tmp = opaque(x); + let out = opaque(tmp); + out +} + +// EMIT_MIR storage.storage_dead_before_return.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn storage_dead_before_return() -> *const u32 { + // This checks that a borrowed local without input storage statements has + // its reconstructed storage ended before returning. + // CHECK-LABEL: fn storage_dead_before_return( + // CHECK: debug x => [[x:_.*]]; + // CHECK: StorageLive([[x]]); + // CHECK: [[x]] = const 1_u32; + // CHECK: [[ret:_.*]] = &raw const [[x]]; + // CHECK: StorageDead([[x]]); + // CHECK-NEXT: return; + mir! { + let x: u32; + debug x => x; + + { + x = 1; + RET = &raw const x; + Return() + } + } +} + +// EMIT_MIR storage.critical_edge_split_for_storage_live.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn critical_edge_split_for_storage_live(flag: bool) { + // This checks storage reconstruction on a custom CFG where both branches + // reach a block which initializes a maybe-live local. The local is dead on + // the direct incoming edge, so inserting StorageLive requires splitting the + // critical edge from the entry switch. + // CHECK-LABEL: fn critical_edge_split_for_storage_live( + // CHECK: debug x => [[crit_tmp:_.*]]; + // CHECK: switchInt(copy _1) -> [1: [[init:bb.*]], otherwise: [[split:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[crit_tmp]]); + // CHECK: [[crit_tmp]] = const 1_u32; + // CHECK: &raw const [[crit_tmp]]; + // CHECK: opaque::<*const u32>{{.*}} -> [return: [[ret:bb.*]], + // CHECK: [[ret]]: { + // CHECK: [[crit_tmp]] = const 2_u32; + // CHECK: [[split]]: { + // CHECK-NEXT: StorageLive([[crit_tmp]]); + // CHECK-NEXT: goto -> [[ret]]; + mir! { + let x: u32; + let ptr: *const u32; + debug x => x; + + { + match flag { + true => init, + _ => ret, + } + } + + init = { + x = 1; + ptr = &raw const x; + Call(ptr = opaque::<*const u32>(ptr), ReturnTo(ret), UnwindUnreachable()) + } + + ret = { + // An initialization forces a StorageLive on both incoming branches, + // which in turn forces a critical edge split. + x = 2; + Return() + } + } +} + +// EMIT_MIR storage.storage_live_elided_on_join_fork.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn storage_live_elided_on_join_fork(flag: bool) { + // This checks a join-fork CFG where x is live in only one predecessor and + // one successor of the middle block. The flag correlation means the path + // which reads x is only reached after x has been initialized. On the direct + // edge to join, every continuation either reads x before initializing it or + // never accesses it, so no StorageLive or critical-edge split is needed. + // CHECK-LABEL: fn storage_live_elided_on_join_fork( + // CHECK: debug x => [[join_tmp:_.*]]; + // CHECK: switchInt(copy _1) -> [1: [[init:bb.*]], otherwise: [[join:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[join_tmp]]); + // CHECK: [[join_tmp]] = const 1_u32; + // CHECK: goto -> [[join]]; + // CHECK: [[join]]: { + // CHECK-NOT: StorageLive([[join_tmp]]); + // CHECK: switchInt(copy _1) -> [1: [[use_x:bb.*]], otherwise: [[done:bb.*]]]; + // CHECK: [[use_x]]: { + // CHECK: opaque::(move [[join_tmp]]) -> [return: [[done]], unwind unreachable]; + mir! { + let x: u32; + let out: u32; + debug x => x; + + { + match flag { + true => init, + _ => join, + } + } + + init = { + x = 1; + Goto(join) + } + + join = { + match flag { + true => use_x, + _ => done, + } + } + + use_x = { + Call(out = opaque::(Move(x)), ReturnTo(done), UnwindUnreachable()) + } + + done = { + Return() + } + } +} diff --git a/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..67a6b0ef87618 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff @@ -0,0 +1,79 @@ +- // MIR for `shorten_non_borrowed` before MoveElimination ++ // MIR for `shorten_non_borrowed` after MoveElimination + + fn shorten_non_borrowed(_1: u32) -> () { + debug x => _1; + let mut _0: (); + let _2: u32; + let mut _3: u32; + let _5: u32; + let mut _6: u32; + let _7: u32; + let mut _8: u32; + scope 1 { +- debug a => _2; ++ debug a => _6; + let _4: u32; + scope 2 { +- debug b => _4; ++ debug b => _6; + } + } + + bb0: { +- StorageLive(_2); ++ nop; ++ nop; + StorageLive(_3); + _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ StorageLive(_6); ++ _6 = opaque::(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_3); +- StorageLive(_4); +- _4 = copy _2; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_5); +- StorageLive(_6); +- _6 = copy _4; + _5 = opaque::(move _6) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_6); + StorageDead(_5); ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_7); +- StorageLive(_8); +- _8 = copy _1; +- _7 = opaque::(move _8) -> [return: bb3, unwind unreachable]; ++ _7 = opaque::(move _1) -> [return: bb3, unwind unreachable]; + } + + bb3: { +- StorageDead(_8); + StorageDead(_7); ++ nop; ++ nop; + _0 = const (); +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.storage_dead_before_return.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.storage_dead_before_return.MoveElimination.diff new file mode 100644 index 0000000000000..bc6299c97b6be --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.storage_dead_before_return.MoveElimination.diff @@ -0,0 +1,17 @@ +- // MIR for `storage_dead_before_return` before MoveElimination ++ // MIR for `storage_dead_before_return` after MoveElimination + + fn storage_dead_before_return() -> *const u32 { + debug x => _1; + let mut _0: *const u32; + let mut _1: u32; + + bb0: { ++ StorageLive(_1); + _1 = const 1_u32; + _0 = &raw const _1; ++ StorageDead(_1); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.storage_live_elided_on_join_fork.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.storage_live_elided_on_join_fork.MoveElimination.diff new file mode 100644 index 0000000000000..d5b9552b4fdf9 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.storage_live_elided_on_join_fork.MoveElimination.diff @@ -0,0 +1,36 @@ +- // MIR for `storage_live_elided_on_join_fork` before MoveElimination ++ // MIR for `storage_live_elided_on_join_fork` after MoveElimination + + fn storage_live_elided_on_join_fork(_1: bool) -> () { + debug x => _2; + let mut _0: (); + let mut _2: u32; + let mut _3: u32; + + bb0: { + switchInt(copy _1) -> [1: bb1, otherwise: bb2]; + } + + bb1: { ++ StorageLive(_2); + _2 = const 1_u32; + goto -> bb2; + } + + bb2: { + switchInt(copy _1) -> [1: bb3, otherwise: bb4]; + } + + bb3: { ++ StorageLive(_3); + _3 = opaque::(move _2) -> [return: bb4, unwind unreachable]; + } + + bb4: { ++ StorageDead(_2); ++ StorageDead(_2); ++ StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff new file mode 100644 index 0000000000000..9a7c3df9cb98c --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff @@ -0,0 +1,63 @@ +- // MIR for `storage_live_moved_to_branch` before MoveElimination ++ // MIR for `storage_live_moved_to_branch` after MoveElimination + + fn storage_live_moved_to_branch(_1: bool) -> () { + debug flag => _1; + let mut _0: (); + let _2: u32; + let mut _3: bool; + let _4: u32; + let mut _5: u32; + scope 1 { +- debug x => _2; ++ debug x => _5; + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- switchInt(move _3) -> [0: bb3, otherwise: bb1]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb1]; + } + + bb1: { +- _2 = const 1_u32; +- StorageLive(_4); + StorageLive(_5); +- _5 = copy _2; ++ _5 = const 1_u32; ++ nop; ++ nop; ++ nop; ++ StorageLive(_4); + _4 = opaque::(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_5); ++ nop; ++ nop; + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); + goto -> bb4; + } + + bb4: { +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff new file mode 100644 index 0000000000000..ffc1f04c58097 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `terminator_end_storage_dead_in_successor` before MoveElimination ++ // MIR for `terminator_end_storage_dead_in_successor` after MoveElimination + + fn terminator_end_storage_dead_in_successor(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + let _2: u32; + let mut _3: u32; + let mut _5: u32; + scope 1 { +- debug tmp => _2; ++ debug tmp => _5; + let _4: u32; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ StorageLive(_5); ++ _5 = opaque::(move _1) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _2; +- _4 = opaque::(move _5) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = opaque::(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { + StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.aggregate.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.aggregate.TailCopyToMove.diff new file mode 100644 index 0000000000000..8c4045d4efcea --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.aggregate.TailCopyToMove.diff @@ -0,0 +1,24 @@ +- // MIR for `aggregate` before TailCopyToMove ++ // MIR for `aggregate` after TailCopyToMove + + fn aggregate(_1: u32, _2: u32) -> Pair { + debug x => _1; + debug y => _2; + let mut _0: Pair; + let mut _3: u32; + let mut _4: u32; + + bb0: { + StorageLive(_3); +- _3 = copy _1; ++ _3 = move _1; + StorageLive(_4); +- _4 = copy _2; ++ _4 = move _2; + _0 = Pair { a: move _3, b: move _4 }; + StorageDead(_4); + StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.aggregate_operands.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.aggregate_operands.TailCopyToMove.diff new file mode 100644 index 0000000000000..cf0c2d62d3960 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.aggregate_operands.TailCopyToMove.diff @@ -0,0 +1,13 @@ +- // MIR for `aggregate_operands` before TailCopyToMove ++ // MIR for `aggregate_operands` after TailCopyToMove + + fn aggregate_operands(_1: u32, _2: u32) -> (u32, u32) { + let mut _0: (u32, u32); + + bb0: { +- _0 = (copy _1, copy _2); ++ _0 = (move _1, move _2); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff new file mode 100644 index 0000000000000..f4a70b1d9fabb --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff @@ -0,0 +1,16 @@ +- // MIR for `aggregate_with_deref` before TailCopyToMove ++ // MIR for `aggregate_with_deref` after TailCopyToMove + + fn aggregate_with_deref(_1: u32) -> (u32, u32) { + let mut _0: (u32, u32); + let mut _2: *const u32; + let mut _3: u32; + + bb0: { + _2 = &raw const _1; + _3 = copy _1; + _0 = (copy _3, copy (*_2)); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff new file mode 100644 index 0000000000000..ac0046e4624e3 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff @@ -0,0 +1,17 @@ +- // MIR for `borrowed_dest_stops_tail` before TailCopyToMove ++ // MIR for `borrowed_dest_stops_tail` after TailCopyToMove + + fn borrowed_dest_stops_tail(_1: u32, _2: u32) -> u32 { + debug y => _3; + let mut _0: u32; + let mut _3: u32; + let mut _4: *const u32; + + bb0: { + _4 = &raw const _3; + _0 = copy _1; + _3 = copy _2; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff new file mode 100644 index 0000000000000..fb24d156a7383 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff @@ -0,0 +1,15 @@ +- // MIR for `borrowed_source_tail` before TailCopyToMove ++ // MIR for `borrowed_source_tail` after TailCopyToMove + + fn borrowed_source_tail(_1: u32) -> u32 { + let mut _0: u32; + let mut _2: *const u32; + + bb0: { + _2 = &raw const _1; +- _0 = copy _1; ++ _0 = move _1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.chain.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.chain.TailCopyToMove.diff new file mode 100644 index 0000000000000..d354f9f23446f --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.chain.TailCopyToMove.diff @@ -0,0 +1,22 @@ +- // MIR for `chain` before TailCopyToMove ++ // MIR for `chain` after TailCopyToMove + + fn chain(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + let _2: u32; + scope 1 { + debug t => _2; + } + + bb0: { + StorageLive(_2); +- _2 = copy _1; +- _0 = copy _2; ++ _2 = move _1; ++ _0 = move _2; + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.direct.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.direct.TailCopyToMove.diff new file mode 100644 index 0000000000000..84e7c4f9d42ca --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.direct.TailCopyToMove.diff @@ -0,0 +1,14 @@ +- // MIR for `direct` before TailCopyToMove ++ // MIR for `direct` after TailCopyToMove + + fn direct(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + + bb0: { +- _0 = copy _1; ++ _0 = move _1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.index_dest.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.index_dest.TailCopyToMove.diff new file mode 100644 index 0000000000000..37ee7527b3492 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.index_dest.TailCopyToMove.diff @@ -0,0 +1,18 @@ +- // MIR for `index_dest` before TailCopyToMove ++ // MIR for `index_dest` after TailCopyToMove + + fn index_dest(_1: [usize; 4], _2: usize) -> [usize; 4] { + debug a => _3; + let mut _0: [usize; 4]; + let mut _3: [usize; 4]; + + bb0: { +- _3 = copy _1; ++ _3 = move _1; + _3[_2] = copy _2; +- _0 = copy _3; ++ _0 = move _3; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.index_operand.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.index_operand.TailCopyToMove.diff new file mode 100644 index 0000000000000..da6bbb117a846 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.index_operand.TailCopyToMove.diff @@ -0,0 +1,13 @@ +- // MIR for `index_operand` before TailCopyToMove ++ // MIR for `index_operand` after TailCopyToMove + + fn index_operand(_1: [u32; 4], _2: usize) -> (usize, u32) { + let mut _0: (usize, u32); + + bb0: { +- _0 = (copy _2, copy _1[_2]); ++ _0 = (copy _2, move _1[_2]); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff new file mode 100644 index 0000000000000..16ac0dd1806ca --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff @@ -0,0 +1,19 @@ +- // MIR for `indirect_tail_read` before TailCopyToMove ++ // MIR for `indirect_tail_read` after TailCopyToMove + + fn indirect_tail_read(_1: u32) -> (u32, u32) { + let mut _0: (u32, u32); + let mut _2: *const u32; + let mut _3: u32; + let mut _4: u32; + + bb0: { + _2 = &raw const _1; + _3 = copy _1; + _4 = copy (*_2); +- _0 = (copy _3, copy _4); ++ _0 = (move _3, move _4); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.indirect_tail_write.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.indirect_tail_write.TailCopyToMove.diff new file mode 100644 index 0000000000000..b98fb6458eec3 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.indirect_tail_write.TailCopyToMove.diff @@ -0,0 +1,16 @@ +- // MIR for `indirect_tail_write` before TailCopyToMove ++ // MIR for `indirect_tail_write` after TailCopyToMove + + fn indirect_tail_write(_1: u32, _2: u32) -> u32 { + debug p => _3; + let mut _0: u32; + let mut _3: *mut u32; + + bb0: { + _3 = &raw mut _1; + _0 = copy _1; + (*_3) = copy _2; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.projected.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.projected.TailCopyToMove.diff new file mode 100644 index 0000000000000..d19a551cfb737 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.projected.TailCopyToMove.diff @@ -0,0 +1,14 @@ +- // MIR for `projected` before TailCopyToMove ++ // MIR for `projected` after TailCopyToMove + + fn projected(_1: Pair) -> u32 { + debug pair => _1; + let mut _0: u32; + + bb0: { +- _0 = copy (_1.0: u32); ++ _0 = move (_1.0: u32); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.projected_dest.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.projected_dest.TailCopyToMove.diff new file mode 100644 index 0000000000000..14911f4313ebd --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.projected_dest.TailCopyToMove.diff @@ -0,0 +1,15 @@ +- // MIR for `projected_dest` before TailCopyToMove ++ // MIR for `projected_dest` after TailCopyToMove + + fn projected_dest(_1: u32, _2: u32) -> (u32, u32) { + let mut _0: (u32, u32); + + bb0: { +- (_0.0: u32) = copy _1; +- (_0.1: u32) = copy _2; ++ (_0.0: u32) = move _1; ++ (_0.1: u32) = move _2; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.repeated_operand.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.repeated_operand.TailCopyToMove.diff new file mode 100644 index 0000000000000..59e775c20f898 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.repeated_operand.TailCopyToMove.diff @@ -0,0 +1,13 @@ +- // MIR for `repeated_operand` before TailCopyToMove ++ // MIR for `repeated_operand` after TailCopyToMove + + fn repeated_operand(_1: u32) -> (u32, u32) { + let mut _0: (u32, u32); + + bb0: { +- _0 = (copy _1, copy _1); ++ _0 = (copy _1, move _1); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.rs b/tests/mir-opt/tail_copy_to_move.rs new file mode 100644 index 0000000000000..c4c18db6570d3 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.rs @@ -0,0 +1,347 @@ +//@ test-mir-pass: TailCopyToMove +//@ compile-flags: -Cpanic=abort + +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +#[derive(Copy, Clone)] +pub struct Pair { + a: u32, + b: u32, +} + +#[derive(Copy, Clone)] +pub enum Choice { + A(u32), + B, +} + +// EMIT_MIR tail_copy_to_move.direct.TailCopyToMove.diff +pub fn direct(x: u32) -> u32 { + // Checks the simplest returned `Copy` local. + // CHECK-LABEL: fn direct( + // CHECK: _0 = move _1; + x +} + +// EMIT_MIR tail_copy_to_move.chain.TailCopyToMove.diff +pub fn chain(x: u32) -> u32 { + // Checks that the scan propagates through a temporary local. + // CHECK-LABEL: fn chain( + // CHECK: debug t => [[TMP:_.*]]; + // CHECK: [[TMP]] = move _1; + // CHECK: _0 = move [[TMP]]; + let t = x; + t +} + +// EMIT_MIR tail_copy_to_move.aggregate.TailCopyToMove.diff +pub fn aggregate(x: u32, y: u32) -> Pair { + // Checks aggregate construction from returned `Copy` locals. + // CHECK-LABEL: fn aggregate( + // CHECK: [[A:_.*]] = move _1; + // CHECK: [[B:_.*]] = move _2; + // CHECK: _0 = Pair { a: move [[A]], b: move [[B]] }; + Pair { a: x, b: y } +} + +// EMIT_MIR tail_copy_to_move.aggregate_operands.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_operands(x: u32, y: u32) -> (u32, u32) { + // Checks aggregate operands that are already in the final assignment. + // CHECK-LABEL: fn aggregate_operands( + // CHECK: _0 = (move _1, move _2); + mir!({ + RET = (x, y); + Return() + }) +} + +// EMIT_MIR tail_copy_to_move.projected_dest.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn projected_dest(x: u32, y: u32) -> (u32, u32) { + // Checks assignments to direct projections of the return place. + // CHECK-LABEL: fn projected_dest( + // CHECK: (_0.0: u32) = move _1; + // CHECK: (_0.1: u32) = move _2; + mir! { + type RET = (u32, u32); + { + RET.0 = x; + RET.1 = y; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.projected.TailCopyToMove.diff +pub fn projected(pair: Pair) -> u32 { + // Checks that direct projected source copies are also rewritten. + // CHECK-LABEL: fn projected( + // CHECK: _0 = move (_1.0: u32); + pair.a +} + +// EMIT_MIR tail_copy_to_move.set_discriminant.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn set_discriminant(choice: Choice) -> Choice { + // Checks that `SetDiscriminant` is accepted in the return tail. + // CHECK-LABEL: fn set_discriminant( + // CHECK: _0 = move _1; + // CHECK: discriminant(_0) = 1; + mir!({ + RET = choice; + SetDiscriminant(RET, 1); + Return() + }) +} + +// EMIT_MIR tail_copy_to_move.set_discriminant_indirect.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn set_discriminant_indirect(choice: Choice) -> Choice { + // Checks that an indirect `SetDiscriminant` place stops the scan. + // CHECK-LABEL: fn set_discriminant_indirect( + // CHECK: debug p => [[P:_.*]]; + // CHECK: _0 = copy _1; + // CHECK: discriminant((*[[P]])) = 1; + mir! { + let p: *mut Choice; + debug p => p; + + { + p = &raw mut choice; + RET = choice; + SetDiscriminant(*p, 1); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.set_discriminant_borrowed.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn set_discriminant_borrowed(input: Choice) -> Choice { + // Checks that writing a borrowed local's discriminant stops the scan. + // CHECK-LABEL: fn set_discriminant_borrowed( + // CHECK: debug local => [[LOCAL:_.*]]; + // CHECK: _0 = copy _1; + // CHECK: discriminant([[LOCAL]]) = 1; + mir! { + let local: Choice; + let p: *const Choice; + debug local => local; + + { + p = &raw const local; + RET = input; + SetDiscriminant(local, 1); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.set_discriminant_index.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn set_discriminant_index(arr: [Choice; 4], idx: usize) -> usize { + // Checks that `SetDiscriminant` records projection locals such as indexes. + // CHECK-LABEL: fn set_discriminant_index( + // CHECK: debug local => [[ARR:_.*]]; + // CHECK: [[ARR]] = move _1; + // CHECK: _0 = copy _2; + // CHECK: discriminant([[ARR]][_2]) = 1; + mir! { + let local: [Choice; 4]; + debug local => local; + + { + local = arr; + RET = idx; + SetDiscriminant(local[idx], 1); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn indirect_tail_read(x: u32) -> (u32, u32) { + // Checks that an indirect read stops the scan before earlier assignments. + // CHECK-LABEL: fn indirect_tail_read( + // CHECK: [[P:_.*]] = &raw const _1; + // CHECK: [[Q:_.*]] = copy _1; + // CHECK: [[S:_.*]] = copy (*[[P]]); + // CHECK: _0 = (move [[Q]], move [[S]]); + mir! { + let p: *const u32; + let q: u32; + let s: u32; + + { + p = &raw const x; + q = x; + s = *p; + RET = (q, s); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.indirect_tail_write.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn indirect_tail_write(x: u32, z: u32) -> u32 { + // Checks that an indirect assignment destination stops the scan. + // CHECK-LABEL: fn indirect_tail_write( + // CHECK: debug p => [[P:_.*]]; + // CHECK: _0 = copy _1; + // CHECK: (*[[P]]) = copy _2; + mir! { + let p: *mut u32; + debug p => p; + + { + p = &raw mut x; + RET = x; + *p = z; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_with_deref(x: u32) -> (u32, u32) { + // Checks that an indirect aggregate operand stops the aggregate scan. + // CHECK-LABEL: fn aggregate_with_deref( + // CHECK: [[P:_.*]] = &raw const _1; + // CHECK: [[Q:_.*]] = copy _1; + // CHECK: _0 = (copy [[Q]], copy (*[[P]])); + mir! { + let p: *const u32; + let q: u32; + + { + p = &raw const x; + q = x; + RET = (q, *p); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn borrowed_dest_stops_tail(x: u32, z: u32) -> u32 { + // Checks that writing to a borrowed local stops the scan. + // CHECK-LABEL: fn borrowed_dest_stops_tail( + // CHECK: debug y => [[Y:_.*]]; + // CHECK: _0 = copy _1; + // CHECK: [[Y]] = copy _2; + mir! { + let y: u32; + let p: *const u32; + debug y => y; + + { + p = &raw const y; + RET = x; + y = z; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn unrelated_tail_store(x: u32, z: u32) -> u32 { + // Checks that writing to an unborrowed local remains in the tail. + // CHECK-LABEL: fn unrelated_tail_store( + // CHECK: debug y => [[Y:_.*]]; + // CHECK: _0 = move _1; + // CHECK: [[Y]] = move _2; + mir! { + let y: u32; + debug y => y; + + { + RET = x; + y = z; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.index_operand.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_operand(arr: [u32; 4], idx: usize) -> (usize, u32) { + // Checks that index projection locals count as later uses. + // CHECK-LABEL: fn index_operand( + // CHECK: _0 = (copy _2, move _1[_2]); + mir! { + { + RET = (idx, arr[idx]); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.index_dest.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_dest(arr: [usize; 4], idx: usize) -> [usize; 4] { + // Checks that index locals in destination projections are recorded. + // CHECK-LABEL: fn index_dest( + // CHECK: debug a => [[ARR:_.*]]; + // CHECK: [[ARR]] = move _1; + // CHECK: [[ARR]][_2] = copy _2; + // CHECK: _0 = move [[ARR]]; + mir! { + let a: [usize; 4]; + debug a => a; + + { + a = arr; + a[idx] = idx; + RET = a; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.repeated_operand.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn repeated_operand(x: u32) -> (u32, u32) { + // Checks right-to-left aggregate scanning for repeated operands. + // CHECK-LABEL: fn repeated_operand( + // CHECK: _0 = (copy _1, move _1); + mir!({ + RET = (x, x); + Return() + }) +} + +// EMIT_MIR tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn borrowed_source_tail(x: u32) -> u32 { + // Checks that a borrowed source can still move at its final use. + // CHECK-LABEL: fn borrowed_source_tail( + // CHECK: [[P:_.*]] = &raw const _1; + // CHECK: _0 = move _1; + mir! { + let p: *const u32; + + { + p = &raw const x; + RET = x; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.shared_return.TailCopyToMove.diff +pub fn shared_return(x: u32, y: u32, take_x: bool) -> u32 { + // Checks branch arms that share a return block. + // CHECK-LABEL: fn shared_return( + // CHECK: _0 = move _1; + // CHECK: _0 = move _2; + if take_x { x } else { y } +} diff --git a/tests/mir-opt/tail_copy_to_move.set_discriminant.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.set_discriminant.TailCopyToMove.diff new file mode 100644 index 0000000000000..76f23dae83a91 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.set_discriminant.TailCopyToMove.diff @@ -0,0 +1,14 @@ +- // MIR for `set_discriminant` before TailCopyToMove ++ // MIR for `set_discriminant` after TailCopyToMove + + fn set_discriminant(_1: Choice) -> Choice { + let mut _0: Choice; + + bb0: { +- _0 = copy _1; ++ _0 = move _1; + discriminant(_0) = 1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.set_discriminant_borrowed.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.set_discriminant_borrowed.TailCopyToMove.diff new file mode 100644 index 0000000000000..945c0e6101b4b --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.set_discriminant_borrowed.TailCopyToMove.diff @@ -0,0 +1,17 @@ +- // MIR for `set_discriminant_borrowed` before TailCopyToMove ++ // MIR for `set_discriminant_borrowed` after TailCopyToMove + + fn set_discriminant_borrowed(_1: Choice) -> Choice { + debug local => _2; + let mut _0: Choice; + let mut _2: Choice; + let mut _3: *const Choice; + + bb0: { + _3 = &raw const _2; + _0 = copy _1; + discriminant(_2) = 1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.set_discriminant_index.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.set_discriminant_index.TailCopyToMove.diff new file mode 100644 index 0000000000000..cde22c5d7a9f9 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.set_discriminant_index.TailCopyToMove.diff @@ -0,0 +1,17 @@ +- // MIR for `set_discriminant_index` before TailCopyToMove ++ // MIR for `set_discriminant_index` after TailCopyToMove + + fn set_discriminant_index(_1: [Choice; 4], _2: usize) -> usize { + debug local => _3; + let mut _0: usize; + let mut _3: [Choice; 4]; + + bb0: { +- _3 = copy _1; ++ _3 = move _1; + _0 = copy _2; + discriminant(_3[_2]) = 1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.set_discriminant_indirect.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.set_discriminant_indirect.TailCopyToMove.diff new file mode 100644 index 0000000000000..80df658ce9e25 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.set_discriminant_indirect.TailCopyToMove.diff @@ -0,0 +1,16 @@ +- // MIR for `set_discriminant_indirect` before TailCopyToMove ++ // MIR for `set_discriminant_indirect` after TailCopyToMove + + fn set_discriminant_indirect(_1: Choice) -> Choice { + debug p => _2; + let mut _0: Choice; + let mut _2: *mut Choice; + + bb0: { + _2 = &raw mut _1; + _0 = copy _1; + discriminant((*_2)) = 1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.shared_return.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.shared_return.TailCopyToMove.diff new file mode 100644 index 0000000000000..26c0879695ae2 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.shared_return.TailCopyToMove.diff @@ -0,0 +1,34 @@ +- // MIR for `shared_return` before TailCopyToMove ++ // MIR for `shared_return` after TailCopyToMove + + fn shared_return(_1: u32, _2: u32, _3: bool) -> u32 { + debug x => _1; + debug y => _2; + debug take_x => _3; + let mut _0: u32; + let mut _4: bool; + + bb0: { + StorageLive(_4); + _4 = copy _3; + switchInt(move _4) -> [0: bb2, otherwise: bb1]; + } + + bb1: { +- _0 = copy _1; ++ _0 = move _1; + goto -> bb3; + } + + bb2: { +- _0 = copy _2; ++ _0 = move _2; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff new file mode 100644 index 0000000000000..4af29766f48e9 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff @@ -0,0 +1,17 @@ +- // MIR for `unrelated_tail_store` before TailCopyToMove ++ // MIR for `unrelated_tail_store` after TailCopyToMove + + fn unrelated_tail_store(_1: u32, _2: u32) -> u32 { + debug y => _3; + let mut _0: u32; + let mut _3: u32; + + bb0: { +- _0 = copy _1; +- _3 = copy _2; ++ _0 = move _1; ++ _3 = move _2; + return; + } + } +