MIR move elimination [1/6]: MIR preparation

compiler/rustc_borrowck/src/dataflow.rs

@@ -530,6 +530,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> {
 
             mir::StatementKind::FakeRead(..)
             | mir::StatementKind::SetDiscriminant { .. }
+            | mir::StatementKind::StorageAlloc(_)
             | mir::StatementKind::StorageLive(..)
             | mir::StatementKind::PlaceMention(..)
             | mir::StatementKind::AscribeUserType(..)

compiler/rustc_borrowck/src/def_use.rs

@@ -76,7 +76,8 @@ pub(crate) fn categorize(context: PlaceContext) -> Option<DefUse> {
         // Backwards incompatible drop hint is not a use, just a marker for linting.
         PlaceContext::NonUse(NonUseContext::BackwardIncompatibleDropHint) => None,
 
-        PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => {
+        PlaceContext::NonUse(NonUseContext::StorageAlloc)
+        | PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => {
             bug!("These statements are not allowed in this MIR phase")
         }
     }

compiler/rustc_borrowck/src/lib.rs

@@ -874,7 +874,9 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a,
                     state,
                 );
             }
-            StatementKind::Nop | StatementKind::SetDiscriminant { .. } => {
+            StatementKind::Nop
+            | StatementKind::StorageAlloc(..)
+            | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
             }
         }

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -83,6 +83,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
             }
             StatementKind::ConstEvalCounter
             | StatementKind::Nop
+            | StatementKind::StorageAlloc(..)
             | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
             }

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -730,6 +730,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             | StatementKind::BackwardIncompatibleDropHint { .. }
             | StatementKind::Nop => {}
             StatementKind::Intrinsic(NonDivergingIntrinsic::CopyNonOverlapping(..))
+            | StatementKind::StorageAlloc(..)
             | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
             }

compiler/rustc_codegen_cranelift/src/base.rs

@@ -614,7 +614,9 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
     fx.set_debug_loc(stmt.source_info);
 
     match &stmt.kind {
-        StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
+        StatementKind::StorageAlloc(..)
+        | StatementKind::StorageLive(..)
+        | StatementKind::StorageDead(..) => {} // Those are not very useful
         _ => {
             if fx.clif_comments.enabled() {
                 let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
@@ -928,7 +930,8 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt:
                 Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"),
             }
         }
-        StatementKind::StorageLive(_)
+        StatementKind::StorageAlloc(_)
+        | StatementKind::StorageLive(_)
         | StatementKind::StorageDead(_)
         | StatementKind::ConstEvalCounter
         | StatementKind::Nop

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -571,6 +571,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         StatementKind::Assign(_)
                         | StatementKind::FakeRead(_)
                         | StatementKind::SetDiscriminant { .. }
+                        | StatementKind::StorageAlloc(_)
                         | StatementKind::StorageLive(_)
                         | StatementKind::StorageDead(_)
                         | StatementKind::AscribeUserType(_, _)

compiler/rustc_codegen_ssa/src/mir/statement.rs

@@ -70,6 +70,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::StatementKind::SetDiscriminant { ref place, variant_index } => {
                 self.codegen_place(bx, (**place).as_ref()).codegen_set_discr(bx, variant_index);
             }
+            mir::StatementKind::StorageAlloc(_) => {
+                // This is a no-op in codegen.
+            }
             mir::StatementKind::StorageLive(local) => {
                 if let LocalRef::Place(cg_place) = self.locals[local] {
                     cg_place.storage_live(bx);

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -707,6 +707,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             StatementKind::Assign(..)
             | StatementKind::SetDiscriminant { .. }
             | StatementKind::FakeRead(..)
+            | StatementKind::StorageAlloc(_)
             | StatementKind::StorageLive(_)
             | StatementKind::StorageDead(_)
             | StatementKind::PlaceMention(..)

compiler/rustc_const_eval/src/interpret/stack.rs

@@ -590,6 +590,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         self.storage_live_dyn(local, MemPlaceMeta::None)
     }
 
+    /// Ensure that a live local has an allocation.
+    pub fn storage_alloc(&mut self, local: mir::Local) -> InterpResult<'tcx> {
+        match self.frame().locals[local].value {
+            LocalValue::Dead => throw_ub!(DeadLocal),
+            LocalValue::Live(_) => interp_ok(()),
+        }
+    }
+
     pub fn storage_dead(&mut self, local: mir::Local) -> InterpResult<'tcx> {
         assert!(local != mir::RETURN_PLACE, "Cannot make return place dead");
         trace!("{:?} is now dead", local);

compiler/rustc_const_eval/src/interpret/step.rs

@@ -109,6 +109,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 self.storage_dead(*local)?;
             }
 
+            // Allocate storage for a local
+            StorageAlloc(local) => {
+                self.storage_alloc(*local)?;
+            }
+
             // No dynamic semantics attached to `FakeRead`; MIR
             // interpreter is solely intended for borrowck'ed code.
             FakeRead(..) => {}

compiler/rustc_index/src/interval.rs

@@ -1,6 +1,7 @@
 use std::iter::Step;
 use std::marker::PhantomData;
-use std::ops::{Bound, Range, RangeBounds};
+use std::ops::{Bound, RangeBounds};
+use std::range::RangeInclusive;
 
 use smallvec::SmallVec;
 
@@ -59,11 +60,14 @@ impl<I: Idx> IntervalSet<I> {
     }
 
     /// Iterates through intervals stored in the set, in order.
-    pub fn iter_intervals(&self) -> impl Iterator<Item = std::ops::Range<I>>
+    pub fn iter_intervals(&self) -> impl Iterator<Item = RangeInclusive<I>>
     where
         I: Step,
     {
-        self.map.iter().map(|&(start, end)| I::new(start as usize)..I::new(end as usize + 1))
+        self.map.iter().map(|&(start, end)| RangeInclusive {
+            start: I::new(start as usize),
+            last: I::new(end as usize),
+        })
     }
 
     /// Returns true if we increased the number of elements present.
@@ -204,17 +208,38 @@ impl<I: Idx> IntervalSet<I> {
         needle <= *prev_end
     }
 
+    /// Returns whether any point in `range` is contained in the set.
+    pub fn intersects_range(&self, range: impl RangeBounds<I> + Clone) -> bool {
+        let start = inclusive_start(range.clone());
+        let Some(end) = inclusive_end(self.domain, range) else {
+            // empty range
+            return false;
+        };
+        if start > end {
+            return false;
+        }
+
+        // Find the last interval whose start is <= end.
+        let Some(last) = self.map.partition_point(|r| r.0 <= end).checked_sub(1) else {
+            // All ranges in the map start after the new range's end
+            return false;
+        };
+        let (_, prev_end) = &self.map[last];
+        start <= *prev_end
+    }
+
     pub fn superset(&self, other: &IntervalSet<I>) -> bool
     where
         I: Step,
     {
         let mut sup_iter = self.iter_intervals();
         let mut current = None;
-        let contains = |sup: Range<I>, sub: Range<I>, current: &mut Option<Range<I>>| {
-            if sup.end < sub.start {
-                // if `sup.end == sub.start`, the next sup doesn't contain `sub.start`
+        let contains = |sup: RangeInclusive<I>,
+                        sub: RangeInclusive<I>,
+                        current: &mut Option<RangeInclusive<I>>| {
+            if sup.last < sub.start {
                 None // continue to the next sup
-            } else if sup.end >= sub.end && sup.start <= sub.start {
+            } else if sup.last >= sub.last && sup.start <= sub.start {
                 *current = Some(sup); // save the current sup
                 Some(true)
             } else {
@@ -224,8 +249,8 @@ impl<I: Idx> IntervalSet<I> {
         other.iter_intervals().all(|sub| {
             current
                 .take()
-                .and_then(|sup| contains(sup, sub.clone(), &mut current))
-                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub.clone(), &mut current)))
+                .and_then(|sup| contains(sup, sub, &mut current))
+                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub, &mut current)))
                 .unwrap_or(false)
         })
     }
@@ -242,11 +267,11 @@ impl<I: Idx> IntervalSet<I> {
             let mut other_current = other_iter.next()?;
 
             loop {
-                if self_current.end <= other_current.start {
+                if self_current.last < other_current.start {
                     self_current = self_iter.next()?;
                     continue;
                 }
-                if other_current.end <= self_current.start {
+                if other_current.last < self_current.start {
                     other_current = other_iter.next()?;
                     continue;
                 }
@@ -374,6 +399,12 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.rows.iter_enumerated()
     }
 
+    pub fn clear_row(&mut self, row: R) {
+        if let Some(row) = self.rows.get_mut(row) {
+            row.clear();
+        }
+    }
+
     fn ensure_row(&mut self, row: R) -> &mut IntervalSet<C> {
         self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size))
     }
@@ -397,6 +428,16 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         write_row.union(read_row)
     }
 
+    pub fn disjoint_rows(&self, a: R, b: R) -> bool
+    where
+        C: Step,
+    {
+        let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else {
+            return true;
+        };
+        a.disjoint(b)
+    }
+
     pub fn insert_all_into_row(&mut self, row: R) {
         self.ensure_row(row).insert_all();
     }

compiler/rustc_index/src/interval/tests.rs

@@ -5,7 +5,7 @@ fn insert_collapses() {
     let mut set = IntervalSet::<u32>::new(10000);
     set.insert_range(9831..=9837);
     set.insert_range(43..=9830);
-    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [43..9838]);
+    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [(43..=9837).into()]);
 }
 
 #[test]

compiler/rustc_middle/src/mir/pretty.rs

@@ -873,6 +873,7 @@ impl Debug for StatementKind<'_> {
             FakeRead((ref cause, ref place)) => {
                 write!(fmt, "FakeRead({cause:?}, {place:?})")
             }
+            StorageAlloc(ref local) => write!(fmt, "StorageAlloc({local:?})"),
             StorageLive(ref place) => write!(fmt, "StorageLive({place:?})"),
             StorageDead(ref place) => write!(fmt, "StorageDead({place:?})"),
             SetDiscriminant { ref place, variant_index } => {

compiler/rustc_middle/src/mir/statement.rs

@@ -57,6 +57,7 @@ impl<'tcx> StatementKind<'tcx> {
             StatementKind::SetDiscriminant { .. } => "SetDiscriminant",
             StatementKind::StorageLive(..) => "StorageLive",
             StatementKind::StorageDead(..) => "StorageDead",
+            StatementKind::StorageAlloc(..) => "StorageAlloc",
             StatementKind::PlaceMention(..) => "PlaceMention",
             StatementKind::AscribeUserType(..) => "AscribeUserType",
             StatementKind::Coverage(..) => "Coverage",

compiler/rustc_middle/src/mir/syntax.rs

@@ -125,7 +125,6 @@ pub enum RuntimePhase {
     /// disallowed:
     /// * [`TerminatorKind::Yield`]
     /// * [`TerminatorKind::CoroutineDrop`]
-    /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
     /// * [`Rvalue::CopyForDeref`]
     /// * [`PlaceElem::OpaqueCast`]
     /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp)
@@ -333,7 +332,11 @@ pub enum StatementKind<'tcx> {
     /// [`Scalar::Initialized`][rustc_abi::Scalar::Initialized]. As a part of this discussion, it is
     /// also unclear in what order the components are evaluated.
     ///
+    /// With `-Zmir-move-elimination`, the rvalue's operands and source places are evaluated before
+    /// the destination place. See [RFC 3943].
+    ///
     /// [#68364]: https://github.com/rust-lang/rust/issues/68364
+    /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943
     ///
     /// See [`Rvalue`] documentation for details on each of those.
     Assign(Box<(Place<'tcx>, Rvalue<'tcx>)>),
@@ -380,11 +383,31 @@ pub enum StatementKind<'tcx> {
     /// If the local is already allocated, calling `StorageLive` again will implicitly free the
     /// local and then allocate fresh uninitialized memory. If a local is already deallocated,
     /// calling `StorageDead` again is a NOP.
+    ///
+    /// With `-Zmir-move-elimination`, `StorageLive` leaves locals live but unallocated. Storage is
+    /// allocated when a destination place directly based on the local is evaluated, or by
+    /// [`StorageAlloc`](StatementKind::StorageAlloc). See [RFC 3943].
+    ///
+    /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943
     StorageLive(Local),
 
     /// See `StorageLive` above.
     StorageDead(Local),
 
+    /// If the local is live but unallocated, allocates backing storage containing uninitialized
+    /// bytes for it. This has no effect if the local already has an allocation.
+    ///
+    /// Calling this on a dead local is UB.
+    ///
+    /// This only has an effect when `-Zmir-move-elimination` is enabled since that is the only way
+    /// to get a live-but-unallocated local. This can be used to ensure a local has storage without
+    /// writing a value, for example before taking its address. See [RFC 3943].
+    ///
+    /// This statement is only permitted in runtime MIR.
+    ///
+    /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943
+    StorageAlloc(Local),
+
     /// This statement exists to preserve a trace of a scrutinee matched against a wildcard binding.
     /// This is especially useful for `let _ = PLACE;` bindings that desugar to a single
     /// `PlaceMention(PLACE)`.
@@ -784,7 +807,13 @@ pub enum TerminatorKind<'tcx> {
     /// The evaluation order is currently "first compute destination place, then `func` operand,
     /// then the arguments in left-to-right order".
     ///
+    /// [RFC 3943] semantics (enabled with -Z mir-move-elimination) changes the evaluation order to
+    /// evaluate the destination place last instead. Additionally, a direct destination must not be
+    /// rooted in a local that is also a whole-local move argument. Violating this restriction makes
+    /// the MIR malformed.
+    ///
     /// [#71117]: https://github.com/rust-lang/rust/issues/71117
+    /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943
     Call {
         /// The function that’s being called.
         func: Operand<'tcx>,
@@ -1323,7 +1352,12 @@ pub enum Operand<'tcx> {
     /// inherently tied to a function call. Are these the semantics we want for MIR? Is this
     /// something we can even decide without knowing more about Rust's memory model?
     ///
+    /// With `-Zmir-move-elimination`, moving a whole local loads its value, frees its allocation,
+    /// and leaves it live but unallocated. This happens during operand evaluation, including for
+    /// call arguments, before evaluating subsequent operands. See [RFC 3943].
+    ///
     /// [UCG#188]: https://github.com/rust-lang/unsafe-code-guidelines/issues/188
+    /// [RFC 3943]: https://github.com/rust-lang/rfcs/pull/3943
     Move(Place<'tcx>),
 
     /// Constants are already semantically values, and remain unchanged.
@@ -1447,9 +1481,6 @@ pub enum Rvalue<'tcx> {
     /// This is needed because dataflow analysis needs to distinguish
     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`
     /// has a destructor.
-    ///
-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After
-    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.
     Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),
 
     /// A CopyForDeref is equivalent to a read from a place at the

compiler/rustc_middle/src/mir/visit.rs

@@ -445,6 +445,13 @@ macro_rules! make_mir_visitor {
                             location
                         );
                     }
+                    StatementKind::StorageAlloc(local) => {
+                        self.visit_local(
+                            $(& $mutability)? *local,
+                            PlaceContext::NonUse(NonUseContext::StorageAlloc),
+                            location
+                        );
+                    }
                     StatementKind::StorageDead(local) => {
                         self.visit_local(
                             $(& $mutability)? *local,
@@ -1365,6 +1372,8 @@ pub enum NonUseContext {
     StorageLive,
     /// Ending a storage live range.
     StorageDead,
+    /// Ensuring a local has allocated storage.
+    StorageAlloc,
     /// User type annotation assertions for NLL.
     AscribeUserTy(ty::Variance),
     /// The data of a user variable, for debug info.
@@ -1464,7 +1473,11 @@ impl PlaceContext {
         match self {
             PlaceContext::MutatingUse(_) => ty::Invariant,
             PlaceContext::NonUse(
-                StorageDead | StorageLive | VarDebugInfo | BackwardIncompatibleDropHint,
+                StorageDead
+                | StorageLive
+                | StorageAlloc
+                | VarDebugInfo
+                | BackwardIncompatibleDropHint,
             ) => ty::Invariant,
             PlaceContext::NonMutatingUse(
                 Inspect | Copy | Move | PlaceMention | SharedBorrow | FakeBorrow | RawBorrow