MIR move elimination [1.5/6]: Ensure ZSTs are always initialized

compiler/rustc_mir_build/src/builder/cfg.rs

@@ -75,11 +75,7 @@ impl<'tcx> CFG<'tcx> {
             source_info,
             place,
             Rvalue::Use(
-                Operand::Constant(Box::new(ConstOperand {
-                    span: source_info.span,
-                    user_ty: None,
-                    const_: Const::zero_sized(tcx.types.unit),
-                })),
+                Operand::zero_sized_constant(tcx.types.unit, source_info.span),
                 WithRetag::Yes,
             ),
         );

compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs

@@ -348,11 +348,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
                 block = this.stmt_expr(block, expr_id, None).into_block();
                 block.and(Rvalue::Use(
-                    Operand::Constant(Box::new(ConstOperand {
-                        span: expr_span,
-                        user_ty: None,
-                        const_: Const::zero_sized(this.tcx.types.unit),
-                    })),
+                    Operand::zero_sized_constant(this.tcx.types.unit, expr_span),
                     WithRetag::Yes,
                 ))
             }

compiler/rustc_mir_build/src/builder/expr/into.rs

@@ -418,8 +418,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     sym::write_via_move => {
                         // `write_via_move(ptr, val)` becomes `*ptr = val` but without any dropping.
 
-                        // The destination must have unit type (so we don't actually have to store anything
-                        // into it).
+                        // The destination must have unit type.
                         assert!(destination.ty(&this.local_decls, this.tcx).ty.is_unit());
 
                         // Compile this to an assignment of the argument into the destination.
@@ -431,7 +430,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             span_bug!(expr_span, "invalid write_via_move call")
                         };
                         let ptr_deref = ptr.project_deeper(&[ProjectionElem::Deref], this.tcx);
-                        this.expr_into_dest(ptr_deref, block, val)
+                        unpack!(block = this.expr_into_dest(ptr_deref, block, val));
+                        this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
+                        block.unit()
                     }
                     sym::write_box_via_move => {
                         // The signature is:

compiler/rustc_mir_transform/src/coroutine/drop.rs

@@ -208,8 +208,18 @@ pub(super) fn create_coroutine_drop_shim<'tcx>(
         }
     }
 
-    // Replace the return variable
+    // Replace the return variable and initialize it once for all return paths.
     body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(tcx.types.unit, source_info);
+    body.basic_blocks_mut()[START_BLOCK].statements.push(Statement::new(
+        source_info,
+        StatementKind::Assign(Box::new((
+            Place::return_place(),
+            Rvalue::Use(
+                Operand::zero_sized_constant(tcx.types.unit, source_info.span),
+                WithRetag::Yes,
+            ),
+        ))),
+    ));
 
     make_coroutine_state_argument_indirect(tcx, &mut body);
 

compiler/rustc_mir_transform/src/coroutine/mod.rs

@@ -749,11 +749,7 @@ fn return_poll_ready_assign<'tcx>(tcx: TyCtxt<'tcx>, source_info: SourceInfo) ->
     // Poll::Ready(())
     let poll_def_id = tcx.require_lang_item(LangItem::Poll, source_info.span);
     let args = tcx.mk_args(&[tcx.types.unit.into()]);
-    let val = Operand::Constant(Box::new(ConstOperand {
-        span: source_info.span,
-        user_ty: None,
-        const_: Const::zero_sized(tcx.types.unit),
-    }));
+    let val = Operand::zero_sized_constant(tcx.types.unit, source_info.span);
     let ready_val = Rvalue::Aggregate(
         Box::new(AggregateKind::Adt(poll_def_id, VariantIdx::from_usize(0), args, None, None)),
         indexvec![val],
@@ -911,6 +907,25 @@ fn create_coroutine_resume_function<'tcx>(
         }
     }
 
+    // Iterator::next has no resume argument, so initialize the unit resume
+    // local at the start of each invocation.
+    if let CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) = transform.coroutine_kind {
+        let source_info = SourceInfo::outermost(body.span);
+        body.basic_blocks_mut()[START_BLOCK].statements.insert(
+            0,
+            Statement::new(
+                source_info,
+                StatementKind::Assign(Box::new((
+                    CTX_ARG.into(),
+                    Rvalue::Use(
+                        Operand::zero_sized_constant(tcx.types.unit, source_info.span),
+                        WithRetag::Yes,
+                    ),
+                ))),
+            ),
+        );
+    }
+
     // Make sure we remove dead blocks to remove
     // unrelated code from the drop part of the function
     simplify::remove_dead_blocks(body);

compiler/rustc_mir_transform/src/cost_checker.rs

@@ -89,6 +89,11 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
                     NonDivergingIntrinsic::CopyNonOverlapping(..) => CALL_PENALTY,
                 };
             }
+            StatementKind::Assign((_, Rvalue::Use(Operand::Constant(ref constant), _)))
+                if matches!(constant.const_, Const::Val(ConstValue::ZeroSized, _)) =>
+            {
+                // Assigning a ZST constant emits no code.
+            }
             StatementKind::Assign(..) => self.penalty += INSTR_COST,
             _ => {}
         }

compiler/rustc_mir_transform/src/instsimplify.rs

@@ -59,7 +59,7 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify {
             ctx.simplify_primitive_clone(terminator, &mut block.statements);
             ctx.simplify_size_or_align_of_val(terminator, &mut block.statements);
             ctx.simplify_raw_eq(terminator, &mut block.statements);
-            ctx.simplify_intrinsic_assert(terminator);
+            ctx.simplify_intrinsic_assert(terminator, &mut block.statements);
             ctx.simplify_nounwind_call(terminator);
             simplify_duplicate_switch_targets(terminator);
         }
@@ -397,9 +397,17 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> {
         }
     }
 
-    fn simplify_intrinsic_assert(&self, terminator: &mut Terminator<'tcx>) {
-        let TerminatorKind::Call { ref func, target: ref mut target @ Some(target_block), .. } =
-            terminator.kind
+    fn simplify_intrinsic_assert(
+        &self,
+        terminator: &mut Terminator<'tcx>,
+        statements: &mut Vec<Statement<'tcx>>,
+    ) {
+        let TerminatorKind::Call {
+            ref func,
+            destination,
+            target: ref mut target @ Some(target_block),
+            ..
+        } = terminator.kind
         else {
             return;
         };
@@ -420,7 +428,21 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> {
                 *target = None;
             }
             Some(false) => {
-                // If we know the assert does not panic, turn the call into a Goto
+                // If we know the assert does not panic, turn the call into a Goto. We still need to
+                // initialize its unit result.
+                statements.push(Statement::new(
+                    terminator.source_info,
+                    StatementKind::Assign(Box::new((
+                        destination,
+                        Rvalue::Use(
+                            Operand::zero_sized_constant(
+                                self.tcx.types.unit,
+                                terminator.source_info.span,
+                            ),
+                            WithRetag::Yes,
+                        ),
+                    ))),
+                ));
                 terminator.kind = TerminatorKind::goto(target_block);
             }
         }

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -47,11 +47,10 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                             StatementKind::Assign(Box::new((
                                 *destination,
                                 Rvalue::Use(
-                                    Operand::Constant(Box::new(ConstOperand {
-                                        span: terminator.source_info.span,
-                                        user_ty: None,
-                                        const_: Const::zero_sized(tcx.types.unit),
-                                    })),
+                                    Operand::zero_sized_constant(
+                                        tcx.types.unit,
+                                        terminator.source_info.span,
+                                    ),
                                     WithRetag::Yes,
                                 ),
                             ))),
@@ -61,20 +60,31 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                     sym::copy_nonoverlapping => {
                         let target = target.unwrap();
                         let Ok([src, dst, count]) = take_array(args) else {
-                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
+                            bug!("Wrong arguments for copy_nonoverlapping intrinsic");
                         };
                         block.statements.push(Statement::new(
                             terminator.source_info,
                             StatementKind::Intrinsic(Box::new(
-                                NonDivergingIntrinsic::CopyNonOverlapping(
-                                    rustc_middle::mir::CopyNonOverlapping {
-                                        src: src.node,
-                                        dst: dst.node,
-                                        count: count.node,
-                                    },
-                                ),
+                                NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
+                                    src: src.node,
+                                    dst: dst.node,
+                                    count: count.node,
+                                }),
                             )),
                         ));
+                        block.statements.push(Statement::new(
+                            terminator.source_info,
+                            StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::Use(
+                                    Operand::zero_sized_constant(
+                                        tcx.types.unit,
+                                        terminator.source_info.span,
+                                    ),
+                                    WithRetag::Yes,
+                                ),
+                            ))),
+                        ));
                         terminator.kind = TerminatorKind::goto(target);
                     }
                     sym::assume => {
@@ -88,6 +98,19 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                                 arg.node,
                             ))),
                         ));
+                        block.statements.push(Statement::new(
+                            terminator.source_info,
+                            StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::Use(
+                                    Operand::zero_sized_constant(
+                                        tcx.types.unit,
+                                        terminator.source_info.span,
+                                    ),
+                                    WithRetag::Yes,
+                                ),
+                            ))),
+                        ));
                         terminator.kind = TerminatorKind::goto(target);
                     }
                     sym::wrapping_add

compiler/rustc_mir_transform/src/match_branches.rs

@@ -65,9 +65,10 @@ impl<'tcx, 'a> SimplifyMatch<'tcx, 'a> {
         otherwise: Option<&ConstOperand<'tcx>>,
     ) -> Option<StatementKind<'tcx>> {
         let (_, first_const, mut others) = split_first_case(consts, otherwise);
-        let first_scalar_int = first_const.const_.try_eval_scalar_int(self.tcx, self.typing_env)?;
+        let first_value =
+            first_const.const_.eval(self.tcx, self.typing_env, first_const.span).ok()?;
         if others.all(|const_| {
-            const_.const_.try_eval_scalar_int(self.tcx, self.typing_env) == Some(first_scalar_int)
+            const_.const_.eval(self.tcx, self.typing_env, const_.span).ok() == Some(first_value)
         }) {
             Some(StatementKind::Assign(Box::new((
                 dest,

compiler/rustc_mir_transform/src/promote_consts.rs

@@ -815,11 +815,10 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                         rhs.clone()
                     } else {
                         let unit = Rvalue::Use(
-                            Operand::Constant(Box::new(ConstOperand {
-                                span: statement.source_info.span,
-                                user_ty: None,
-                                const_: Const::zero_sized(self.tcx.types.unit),
-                            })),
+                            Operand::zero_sized_constant(
+                                self.tcx.types.unit,
+                                statement.source_info.span,
+                            ),
                             WithRetag::Yes,
                         );
                         mem::replace(rhs, unit)

compiler/rustc_mir_transform/src/remove_zsts.rs

@@ -127,17 +127,24 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
                 Some(place)
             }
             StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
-                Some(local.into())
+                if self.known_to_be_zst(self.local_decls[local].ty) {
+                    statement.make_nop(true);
+                }
+                return;
             }
             StatementKind::Coverage(_)
             | StatementKind::Intrinsic(_)
             | StatementKind::Nop
             | StatementKind::BackwardIncompatibleDropHint { .. }
             | StatementKind::ConstEvalCounter => None,
         };
+        // Under move-elimination semantics, direct ZST writes still serve to
+        // allocate the base local and give it an address. Genuinely unused
+        // assignments are later deleted by DSE.
         if let Some(place_for_ty) = place_for_ty
             && let ty = place_for_ty.ty(self.local_decls, self.tcx).ty
             && self.known_to_be_zst(ty)
+            && place_for_ty.is_indirect()
         {
             statement.make_nop(true);
         } else {

compiler/rustc_mir_transform/src/shim.rs

@@ -278,13 +278,20 @@ pub fn build_drop_shim<'tcx>(
 
     let return_block = BasicBlock::new(1);
     let mut blocks = IndexVec::with_capacity(2);
-    let block = |blocks: &mut IndexVec<_, _>, kind| {
+    let block = |blocks: &mut IndexVec<_, _>, kind| -> BasicBlock {
         blocks.push(BasicBlockData::new(Some(Terminator { source_info, kind }), false))
     };
     if ty.is_some() {
         block(&mut blocks, TerminatorKind::goto(return_block));
     }
-    block(&mut blocks, TerminatorKind::Return);
+    let ret = block(&mut blocks, TerminatorKind::Return);
+    blocks[ret].statements.push(Statement::new(
+        source_info,
+        StatementKind::Assign(Box::new((
+            Place::return_place(),
+            Rvalue::Use(Operand::zero_sized_constant(tcx.types.unit, span), WithRetag::Yes),
+        ))),
+    ));
 
     let source = MirSource::from_shim(ty::ShimKind::DropGlue(def_id, ty));
     let mut body =
@@ -699,10 +706,18 @@ impl<'tcx> CloneShimBuilder<'tcx> {
         unwind
     }
 
-    fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
-    where
-        I: IntoIterator<Item = Ty<'tcx>>,
-    {
+    fn tuple_like_shim(
+        &mut self,
+        dest: Place<'tcx>,
+        src: Place<'tcx>,
+        tys: &'tcx ty::List<Ty<'tcx>>,
+    ) {
+        if tys.is_empty() {
+            // With no fields to clone, initialize the return local directly.
+            self.copy_shim();
+            return;
+        }
+
         self.block(vec![], TerminatorKind::goto(self.block_index_offset(3)), false);
         let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
         let target = self.block(vec![], TerminatorKind::Return, false);

compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs

@@ -84,7 +84,7 @@ pub(super) fn build_async_drop_shim<'tcx>(
 
     let return_block = BasicBlock::new(1);
     let mut blocks = IndexVec::with_capacity(2);
-    let block = |blocks: &mut IndexVec<_, _>, kind| {
+    let block = |blocks: &mut IndexVec<_, _>, kind| -> BasicBlock {
         blocks.push(BasicBlockData::new(Some(Terminator { source_info, kind }), false))
     };
     block(
@@ -101,7 +101,14 @@ pub(super) fn build_async_drop_shim<'tcx>(
             TerminatorKind::goto(return_block)
         },
     );
-    block(&mut blocks, TerminatorKind::Return);
+    let ret = block(&mut blocks, TerminatorKind::Return);
+    blocks[ret].statements.push(Statement::new(
+        source_info,
+        StatementKind::Assign(Box::new((
+            Place::return_place(),
+            Rvalue::Use(Operand::zero_sized_constant(tcx.types.unit, span), WithRetag::Yes),
+        ))),
+    ));
 
     let source = MirSource::from_shim(ty::ShimKind::AsyncDropGlue(def_id, ty));
     let mut body =