diff --git a/compiler/rustc_attr_ir/src/data_structures.rs b/compiler/rustc_attr_ir/src/data_structures.rs index bca4d8fa7b5ff..04fee06e2434c 100644 --- a/compiler/rustc_attr_ir/src/data_structures.rs +++ b/compiler/rustc_attr_ir/src/data_structures.rs @@ -589,6 +589,7 @@ pub enum RustcDumpLayoutKind { #[derive(Clone, Debug, StableHash, Encodable, Decodable, PrintAttribute, PartialEq, Eq)] pub enum RustcMirKind { PrettyLiveLocals, + PrettyPreciseLiveness, PrettyTransitiveLiveLocals, PeekMaybeInit, PeekMaybeUninit, diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs index b9c03883d7c0b..70e5e4803c362 100644 --- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs +++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs @@ -672,6 +672,10 @@ impl CombineAttributeParser for RustcMirParser { cx.expect_no_args(mi.args())?; Some(RustcMirKind::PrettyLiveLocals) } + sym::rustc_pretty_precise_liveness => { + cx.expect_no_args(mi.args())?; + Some(RustcMirKind::PrettyPreciseLiveness) + } sym::rustc_pretty_transitive_live_locals => { cx.expect_no_args(mi.args())?; Some(RustcMirKind::PrettyTransitiveLiveLocals) diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index 7f46b19f7568f..e2446d94993a7 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -16,7 +16,7 @@ use rustc_abi::{CanonAbi, ExternAbi, X86Call}; use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization; use rustc_codegen_ssa::diagnostics::CompilerBuiltinsCannotCall; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv as _}; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ShimKind, TypeVisitableExt}; use rustc_session::Session; @@ -441,7 +441,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( let instance = if let ty::FnDef(def_id, fn_args) = *func.layout().ty.kind() { let instance = ty::Instance::expect_resolve( fx.tcx, - ty::TypingEnv::fully_monomorphized(), + fx.typing_env(), def_id, fn_args.no_bound_vars().unwrap(), source_info.span, diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index a6d5e4c6a6423..6828c6cd0ab4f 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -720,7 +720,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let func_ref = fx.get_function_ref( Instance::resolve_for_fn_ptr( fx.tcx, - ty::TypingEnv::fully_monomorphized(), + fx.typing_env(), def_id, args.no_bound_vars().unwrap(), ) @@ -771,8 +771,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: fn is_wide_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { ty.builtin_deref(true).is_some_and(|pointee_ty| { - fx.tcx - .type_has_metadata(pointee_ty, ty::TypingEnv::fully_monomorphized()) + fx.tcx.type_has_metadata(pointee_ty, fx.typing_env()) }) } diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index 94a71c65ce261..d59fb0f93ed4f 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -4,8 +4,8 @@ use rustc_abi::{Float, Integer, Primitive}; use rustc_index::IndexVec; use rustc_middle::ty::TypeFoldable; use rustc_middle::ty::layout::{ - self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, - codegen_handle_fn_abi_err, + self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv as _, LayoutError, + LayoutOfHelpers, codegen_handle_fn_abi_err, }; use rustc_span::Symbol; use rustc_target::callconv::FnAbi; @@ -351,7 +351,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { { self.instance.instantiate_mir_and_normalize_erasing_regions( self.tcx, - ty::TypingEnv::fully_monomorphized(), + self.typing_env(), ty::EarlyBinder::bind(self.tcx, value), ) } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index f1e4fd818ebeb..5fc02a19c0094 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -9,6 +9,7 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::{ AllocId, GlobalAlloc, PointerArithmetic, Scalar, read_target_uint, }; +use rustc_middle::ty::layout::HasTypingEnv as _; use rustc_middle::ty::{ExistentialTraitRef, ScalarInt}; use crate::prelude::*; @@ -81,7 +82,7 @@ pub(crate) fn eval_mir_constant<'tcx>( let cv = fx.monomorphize(constant.const_); // This cannot fail because we checked all required_consts in advance. let val = cv - .eval(fx.tcx, ty::TypingEnv::fully_monomorphized(), constant.span) + .eval(fx.tcx, fx.typing_env(), constant.span) .expect("erroneous constant missed by mono item collection"); (val, cv.ty()) } diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index b2bf856e7bcee..c13af35f94108 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -7,7 +7,7 @@ use rustc_abi::CanonAbi; use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_attr_ir::lang_items::LangItem; use rustc_middle::mir::interpret::{GlobalAlloc, PointerArithmetic, Scalar as ConstScalar}; -use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv as _}; use rustc_span::sym; use rustc_target::asm::*; use rustc_target::spec::Arch; @@ -175,7 +175,7 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>( if let ty::FnDef(def_id, args) = *const_.ty().kind() { let instance = ty::Instance::resolve_for_fn_ptr( fx.tcx, - ty::TypingEnv::fully_monomorphized(), + fx.typing_env(), def_id, args.no_bound_vars().unwrap(), ) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 9c3182c944dc6..1437ea7445d24 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -22,7 +22,7 @@ use cranelift_codegen::ir::{ }; use rustc_middle::ty; use rustc_middle::ty::GenericArgsRef; -use rustc_middle::ty::layout::ValidityRequirement; +use rustc_middle::ty::layout::{HasTypingEnv as _, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_span::{Spanned, Symbol, sym}; use rustc_target::spec::PanicStrategy; @@ -729,10 +729,7 @@ fn codegen_regular_intrinsic_call<'tcx>( if let Some(requirement) = requirement { let do_panic = !fx .tcx - .check_validity_requirement(( - requirement, - ty::TypingEnv::fully_monomorphized().as_query_input(ty), - )) + .check_validity_requirement((requirement, fx.typing_env().as_query_input(ty))) .expect("expect to have layout during codegen"); if do_panic { diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs index f1c44df1f6886..317e61031b34c 100644 --- a/compiler/rustc_codegen_cranelift/src/num.rs +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -1,5 +1,7 @@ //! Various operations on integer and floating-point numbers +use rustc_middle::ty::layout::HasTypingEnv as _; + use crate::codegen_f16_f128; use crate::prelude::*; @@ -437,7 +439,7 @@ fn codegen_ptr_binop<'tcx>( .layout() .ty .builtin_deref(true) - .map(|ty| !fx.tcx.type_has_metadata(ty, ty::TypingEnv::fully_monomorphized())) + .map(|ty| !fx.tcx.type_has_metadata(ty, fx.typing_env())) .unwrap_or(true); if is_thin_ptr { diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index 440ae9c4b812e..a4af73cbc3f20 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -693,7 +693,7 @@ impl<'tcx> CPlace<'tcx> { }; let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field); - if fx.tcx.type_has_metadata(field_layout.ty, ty::TypingEnv::fully_monomorphized()) { + if fx.tcx.type_has_metadata(field_layout.ty, fx.typing_env()) { CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout) } else { CPlace::for_ptr(field_ptr, field_layout) @@ -808,7 +808,7 @@ impl<'tcx> CPlace<'tcx> { pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> { let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap()); - if fx.tcx.type_has_metadata(inner_layout.ty, ty::TypingEnv::fully_monomorphized()) { + if fx.tcx.type_has_metadata(inner_layout.ty, fx.typing_env()) { let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx); CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout) } else { @@ -821,7 +821,7 @@ impl<'tcx> CPlace<'tcx> { fx: &mut FunctionCx<'_, '_, 'tcx>, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - if fx.tcx.type_has_metadata(self.layout().ty, ty::TypingEnv::fully_monomorphized()) { + if fx.tcx.type_has_metadata(self.layout().ty, fx.typing_env()) { let (ptr, extra) = self.to_ptr_unsized(); CValue::by_val_pair(ptr.get_addr(fx), extra, layout) } else { diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs index fb86b8a7f905e..902f8ba52faaa 100644 --- a/compiler/rustc_codegen_gcc/src/consts.rs +++ b/compiler/rustc_codegen_gcc/src/consts.rs @@ -16,8 +16,8 @@ use rustc_middle::mir::interpret::{ self, ConstAllocation, CtfeProvenance, ErrorHandled, Scalar as InterpScalar, read_target_uint, }; use rustc_middle::mono::MonoItem; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::{self, Instance}; +use rustc_middle::ty::Instance; +use rustc_middle::ty::layout::{HasTypingEnv as _, LayoutOf}; use rustc_span::def_id::DefId; use rustc_span::{bug, span_bug}; @@ -251,7 +251,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let gcc_type = if nested { self.type_i8() } else { - let ty = instance.ty(self.tcx, ty::TypingEnv::fully_monomorphized()); + let ty = instance.ty(self.tcx, self.typing_env()); self.layout_of(ty).gcc_type(self) }; diff --git a/compiler/rustc_codegen_gcc/src/int.rs b/compiler/rustc_codegen_gcc/src/int.rs index de8762d1c06b4..eaf44c35ded01 100644 --- a/compiler/rustc_codegen_gcc/src/int.rs +++ b/compiler/rustc_codegen_gcc/src/int.rs @@ -10,7 +10,8 @@ use gccjit::{ use rustc_abi::{CanonAbi, Endian, ExternAbi}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::traits::{BackendTypes, BaseTypeCodegenMethods, BuilderMethods, OverflowOp}; -use rustc_middle::ty::{self, Ty}; +use rustc_middle::ty::Ty; +use rustc_middle::ty::layout::HasTypingEnv as _; use rustc_target::callconv::{ArgAbi, ArgAttributes, FnAbi, PassMode}; use rustc_type_ir::{Interner, TyKind}; @@ -387,10 +388,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { 128 => self.tcx.types.i128, _ => unreachable!("unexpected integer size"), }; - let layout = self - .tcx - .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(res_ty)) - .unwrap(); + let layout = self.tcx.layout_of(self.cx.typing_env().as_query_input(res_ty)).unwrap(); let arg_abi = ArgAbi { layout, mode: PassMode::Direct(ArgAttributes::new()) }; let mut fn_abi = FnAbi { diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs index 54013b6be6173..959ad826359f9 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs @@ -16,7 +16,7 @@ use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, LayoutTy use rustc_hir as hir; use rustc_middle::mir::BinOp; use rustc_middle::ty::consts::ConstExt; -use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv as _, LayoutOf}; use rustc_middle::ty::{self, Ty}; use rustc_span::{ErrorGuaranteed, Span, Symbol, span_bug, sym}; @@ -540,7 +540,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( match *in_elem.kind() { ty::RawPtr(p_ty, _) => { let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { - bx.tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), ty) + bx.tcx.normalize_erasing_regions(bx.typing_env(), ty) }); require!( metadata.is_unit(), @@ -554,7 +554,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( match *out_elem.kind() { ty::RawPtr(p_ty, _) => { let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| { - bx.tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), ty) + bx.tcx.normalize_erasing_regions(bx.typing_env(), ty) }); require!( metadata.is_unit(), diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index d52e34d30fb7d..b171424b46245 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -312,11 +312,7 @@ fn fat_lto( let data = bc_decoded.data(); unsafe { - if !llvm::LLVMRustLinkerAdd( - linker, - data.as_ptr() as *const libc::c_char, - data.len(), - ) { + if !llvm::LLVMRustLinkerAdd(linker, data.as_ptr(), data.len()) { llvm::LLVMRustLinkerFree(linker); write::llvm_err(dcx, LlvmError::LoadBitcode { name }) } @@ -327,10 +323,9 @@ fn fat_lto( // Internalize everything below threshold to help strip out more modules and such. unsafe { - let ptr = symbols_below_threshold.as_ptr(); llvm::LLVMRustRunRestrictionPass( llmod, - ptr as *const *const libc::c_char, + symbols_below_threshold.as_ptr(), symbols_below_threshold.len() as libc::size_t, ); } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 03058ba02d9dc..3b1e36265b941 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -10,7 +10,7 @@ use rustc_codegen_ssa::mir::place::PlaceValue; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot}; use rustc_data_structures::thin_vec::ThinVec; use rustc_middle::ty; -use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; +use rustc_middle::ty::{Ty, TyCtxt, TypingEnv}; use rustc_span::bug; use rustc_target::callconv::PassMode; use tracing::debug; @@ -50,12 +50,8 @@ pub(crate) fn adjust_activity_to_abi<'tcx>( if let ty::Slice(element_ty) = tail_ty.kind() { // Now we need to figure out the size of each slice element in memory to allow // safety checks and usability improvements in the backend. - let pci = PseudoCanonicalInput { - typing_env: TypingEnv::fully_monomorphized(), - value: *element_ty, - }; - let layout = tcx.layout_of(pci); + let layout = tcx.layout_of(typing_env.as_query_input(*element_ty)); let elem_size = match layout { Ok(layout) => layout.size, Err(_) => { @@ -91,9 +87,7 @@ pub(crate) fn adjust_activity_to_abi<'tcx>( } } - let pci = PseudoCanonicalInput { typing_env: TypingEnv::fully_monomorphized(), value: *ty }; - - let layout = match tcx.layout_of(pci) { + let layout = match tcx.layout_of(typing_env.as_query_input(*ty)) { Ok(layout) => layout.layout, Err(_) => { bug!("failed to compute layout for type {:?}", ty); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index a5a4f0b3ddb25..651082cee5d43 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -470,20 +470,14 @@ pub(crate) fn val_ty(v: &Value) -> &Type { } pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { - unsafe { - let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), TRUE) - } + unsafe { llvm::LLVMConstStringInContext2(llcx, bytes.as_ptr(), bytes.len(), TRUE) } } pub(crate) fn null_terminate_bytes_in_context<'ll>( llcx: &'ll llvm::Context, bytes: &[u8], ) -> &'ll Value { - unsafe { - let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), FALSE) - } + unsafe { llvm::LLVMConstStringInContext2(llcx, bytes.as_ptr(), bytes.len(), FALSE) } } pub(crate) fn named_struct<'ll>(ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 1059de3c0aec0..46196ab7d360f 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1,6 +1,6 @@ use std::borrow::{Borrow, Cow}; use std::cell::{Cell, RefCell}; -use std::ffi::{CStr, c_char, c_uint}; +use std::ffi::{CStr, c_uint}; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::str; @@ -911,19 +911,11 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } pub(crate) fn get_md_kind_id(&self, name: &str) -> llvm::MetadataKindId { - unsafe { - llvm::LLVMGetMDKindIDInContext( - self.llcx(), - name.as_ptr() as *const c_char, - name.len() as c_uint, - ) - } + unsafe { llvm::LLVMGetMDKindIDInContext(self.llcx(), name.as_ptr(), name.len() as c_uint) } } pub(crate) fn create_metadata(&self, name: &[u8]) -> &'ll Metadata { - unsafe { - llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len()) - } + unsafe { llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr(), name.len()) } } pub(crate) fn get_functions(&self) -> Vec<&'ll Value> { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 0e7547a0db9a4..bea61bd79f561 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -22,7 +22,7 @@ use rustc_middle::mir::BinOp; use rustc_middle::ty::consts::ConstExt; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; use rustc_middle::ty::offload_meta::OffloadMetadata; -use rustc_middle::ty::{self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt, TypingEnv}; +use rustc_middle::ty::{self, GenericArgsRef, Instance, SimdAlign, Ty, TyCtxt}; use rustc_session::diagnostics::feature_err; use rustc_span::{ErrorGuaranteed, Span, Symbol, bug, span_bug, sym}; use rustc_structures::CrateType; @@ -1886,7 +1886,7 @@ fn codegen_autodiff<'ll, 'tcx>( adjust_activity_to_abi( tcx, source_fn_ptr_ty, - TypingEnv::fully_monomorphized(), + bx.cx.typing_env(), &mut diff_attrs.input_activity, ); diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 68e5ed3d37741..2e76223c68c12 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -329,10 +329,7 @@ pub(crate) mod Enzyme_AD { let c_fun_name = std::ffi::CString::new(fun_name) .unwrap_or_else(|err| bug!("failed to set_print_type_fun: {err}")); unsafe { - (self.EnzymeSetCLString)( - self.EnzymeFunctionToAnalyze, - c_fun_name.as_ptr() as *const c_char, - ); + (self.EnzymeSetCLString)(self.EnzymeFunctionToAnalyze, c_fun_name.as_ptr()); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 7e3f2699fff0b..749ef8120da6f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -890,7 +890,7 @@ unsafe extern "C" { pub(crate) fn LLVMContextSetDiscardValueNames(C: &Context, Discard: Bool); pub(crate) fn LLVMGetMDKindIDInContext( C: &Context, - Name: *const c_char, + Name: *const c_uchar, // See "PTR_LEN_STR". SLen: c_uint, ) -> MetadataKindId; @@ -1009,7 +1009,7 @@ unsafe extern "C" { // Operations on metadata pub(crate) fn LLVMMDStringInContext2( C: &Context, - Str: *const c_char, + Str: *const c_uchar, // See "PTR_LEN_STR". SLen: size_t, ) -> &Metadata; pub(crate) fn LLVMMDNodeInContext2<'a>( @@ -1042,7 +1042,7 @@ unsafe extern "C" { pub(crate) fn LLVMArrayType2(ElementType: &Type, ElementCount: u64) -> &Type; pub(crate) fn LLVMConstStringInContext2( C: &Context, - Str: *const c_char, + Str: *const c_uchar, // See "PTR_LEN_STR". Length: size_t, DontNullTerminate: Bool, ) -> &Value; @@ -2561,7 +2561,7 @@ unsafe extern "C" { pub(crate) fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; pub(crate) fn LLVMRustLinkerAdd( linker: &Linker<'_>, - bytecode: *const c_char, + bytecode: *const c_uchar, // See "PTR_LEN_STR". bytecode_len: usize, ) -> bool; pub(crate) fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index f9dfd04faf2ac..e7bdb30c370dc 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -451,7 +451,7 @@ where let instance = match ty.kind() { &ty::FnDef(def_id, args) => Instance::expect_resolve( cx.tcx(), - ty::TypingEnv::fully_monomorphized(), + cx.typing_env(), def_id, args.no_bound_vars().unwrap(), expr.span, diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 8e3533deb5dfc..a6ad7282ca08d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -222,7 +222,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if tcx.features().ergonomic_clones() { let monomorphized_mir = instance.instantiate_mir_and_normalize_erasing_regions( tcx, - ty::TypingEnv::fully_monomorphized(), + cx.typing_env(), ty::EarlyBinder::bind(tcx, mir.clone()), ); mir = tcx.arena.alloc(optimize_use_clone::(cx, monomorphized_mir)); diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 1b99f54719fb4..2a03a840eb1aa 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -449,9 +449,16 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> { // all. That said, we have to allow calling functions inside a `const trait`. These // *are* const-checked! if !ecx.tcx.is_const_fn(def) || find_attr!(ecx.tcx, def, RustcDoNotConstCheck) { - // We certainly do *not* want to actually call the fn - // though, so be sure we return here. - throw_unsup_format!("calling non-const function `{}`", instance) + // We certainly do *not* want to actually call this fn. const-checking should have + // prevented this, but this can be reached with unleashed-miri, so make testing + // simpler by avoiding ICEs there. (Technically this can still ICE if miri-unleash + // was set in a dependency rather than the current crate. That's okay, it's an + // internal unstable feature.) + if ecx.tcx.sess.opts.unstable_opts.unleash_the_miri_inside_of_you { + throw_unsup_format!("calling non-const function `{}`", instance) + } else { + span_bug!(ecx.cur_span(), "calling non-const function `{}`", instance) + } } } diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs index 98155d9a8d5ef..fbe20c849de9c 100644 --- a/compiler/rustc_hir_typeck/src/upvar.rs +++ b/compiler/rustc_hir_typeck/src/upvar.rs @@ -1867,7 +1867,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } // Anything else would be completely captured and therefore handled already. - _ => unreachable!(), + _ => unreachable!("unexpected base_path_ty: {base_path_ty:?}",), } } diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs index 3a245a5b25759..a0c8eb22a8273 100644 --- a/compiler/rustc_infer/src/infer/canonical/query_response.rs +++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs @@ -108,7 +108,7 @@ impl<'tcx> InferCtxt<'tcx> { }; self.canonicalize_response(QueryResponse { - var_values: inference_vars, + var_values: inference_vars.var_values, region_constraints: QueryRegionConstraints::default(), certainty: Certainty::Proven, // Ambiguities are OK! opaque_types, @@ -164,7 +164,7 @@ impl<'tcx> InferCtxt<'tcx> { .collect(); Ok(QueryResponse { - var_values: inference_vars, + var_values: inference_vars.var_values, region_constraints, certainty, value: answer, @@ -282,9 +282,8 @@ impl<'tcx> InferCtxt<'tcx> { for (index, original_value) in original_values.var_values.iter().enumerate() { // ...with the value `v_r` of that variable from the query. - let result_value = query_response.instantiate_projected(self.tcx, &result_args, |v| { - v.var_values[BoundVar::new(index)] - }); + let result_value = query_response + .instantiate_projected(self.tcx, &result_args, |v| v.var_values[index]); match (original_value.kind(), result_value.kind()) { (GenericArgKind::Lifetime(re1), GenericArgKind::Lifetime(re2)) if re1.is_erased() && re2.is_erased() => @@ -435,7 +434,7 @@ impl<'tcx> InferCtxt<'tcx> { // the inputs to the query. Therefore, we begin by unifying // these values with the original inputs that were // canonicalized. - let result_values = &query_response.value.var_values; + let result_values = query_response.value.var_values; assert_eq!(original_values.var_values.len(), result_values.len()); // Quickly try to find initial values for the canonical @@ -557,7 +556,7 @@ impl<'tcx> InferCtxt<'tcx> { // canonical variable; this is taken from // `query_response.var_values` after applying the instantiation // by `result_args`. - let instantiated_query_response = |index: BoundVar| -> GenericArg<'tcx> { + let instantiated_query_response = |index: usize| -> GenericArg<'tcx> { query_response.instantiate_projected(self.tcx, result_args, |v| v.var_values[index]) }; @@ -573,11 +572,11 @@ impl<'tcx> InferCtxt<'tcx> { cause: &ObligationCause<'tcx>, param_env: ty::ParamEnv<'tcx>, variables1: &OriginalQueryValues<'tcx>, - variables2: impl Fn(BoundVar) -> GenericArg<'tcx>, + variables2: impl Fn(usize) -> GenericArg<'tcx>, ) -> InferResult<'tcx, ()> { let mut obligations = PredicateObligations::new(); for (index, value1) in variables1.var_values.iter().enumerate() { - let value2 = variables2(BoundVar::new(index)); + let value2 = variables2(index); match (value1.kind(), value2.kind()) { (GenericArgKind::Type(v1), GenericArgKind::Type(v2)) => { diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs index 46429f7adfb12..291a7b8e59f4f 100644 --- a/compiler/rustc_middle/src/infer/canonical.rs +++ b/compiler/rustc_middle/src/infer/canonical.rs @@ -30,7 +30,7 @@ pub use rustc_type_ir as ir; use smallvec::SmallVec; use crate::mir::ConstraintCategory; -use crate::ty::{self, GenericArg, List, Ty, TyCtxt, TypeFlags, TypeVisitableExt}; +use crate::ty::{self, GenericArg, GenericArgsRef, List, Ty, TyCtxt, TypeFlags, TypeVisitableExt}; pub type CanonicalQueryInput<'tcx, V> = ir::CanonicalQueryInput, V>; pub type Canonical<'tcx, V> = ir::Canonical, V>; @@ -69,7 +69,7 @@ impl<'tcx> Default for OriginalQueryValues<'tcx> { /// `instantiate_query_result` to access the data in this result. #[derive(Clone, Debug, StableHash, TypeFoldable, TypeVisitable)] pub struct QueryResponse<'tcx, R> { - pub var_values: CanonicalVarValues<'tcx>, + pub var_values: GenericArgsRef<'tcx>, pub region_constraints: QueryRegionConstraints<'tcx>, pub certainty: Certainty, pub opaque_types: Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>, diff --git a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs index 7fcabd423e778..c7837eeec307b 100644 --- a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs +++ b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs @@ -74,10 +74,14 @@ impl<'tcx> TyCtxt<'tcx> { ) { let value = self.erase_and_anonymize_regions(value); if value.has_aliases() { - assert_eq!( - value.clone(), - value.fold_with(&mut NormalizeAfterErasingRegionsFolder { tcx: self, typing_env }) - ) + // We want this check to also detect cases where an alias is incorrectly + // rigid in the current `TypingEnv`. This is the case if we use values + // which have been normalized in a different `TypingEnv` without + // renormalizing them. This was necessary to catch e.g. #163724. + let normalized = ty::set_aliases_to_non_rigid(self, value.clone()) + .skip_normalization() + .fold_with(&mut NormalizeAfterErasingRegionsFolder { tcx: self, typing_env }); + assert_eq!(value, normalized); } } diff --git a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs index 09c76d6beb4a1..18dea6dafd2dd 100644 --- a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs +++ b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs @@ -736,7 +736,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // Helper to get a `-1` value of the appropriate type fn neg_1_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> { - let typing_env = ty::TypingEnv::fully_monomorphized(); + let typing_env = self.typing_env(); let size = self.tcx.layout_of(typing_env.as_query_input(ty)).unwrap().size; let literal = Const::from_bits(self.tcx, size.unsigned_int_max(), typing_env, ty); @@ -746,7 +746,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // Helper to get the minimum value of the appropriate type fn minval_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> { assert!(ty.is_signed()); - let typing_env = ty::TypingEnv::fully_monomorphized(); + let typing_env = self.typing_env(); let bits = self.tcx.layout_of(typing_env.as_query_input(ty)).unwrap().size.bits(); let n = 1 << (bits - 1); let literal = Const::from_bits(self.tcx, n, typing_env, ty); diff --git a/compiler/rustc_mir_build/src/builder/misc.rs b/compiler/rustc_mir_build/src/builder/misc.rs index 6e8e74fd4fc8d..101d07cea2ce7 100644 --- a/compiler/rustc_mir_build/src/builder/misc.rs +++ b/compiler/rustc_mir_build/src/builder/misc.rs @@ -2,7 +2,7 @@ //! kind of thing. use rustc_middle::mir::*; -use rustc_middle::ty::{self, Ty}; +use rustc_middle::ty::Ty; use rustc_span::Span; use rustc_trait_selection::infer::InferCtxtExt; use tracing::debug; @@ -32,7 +32,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { /// Returns a zero literal operand for the appropriate type, works for /// bool, char and integers. pub(crate) fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> { - let literal = Const::from_bits(self.tcx, 0, ty::TypingEnv::fully_monomorphized(), ty); + let literal = Const::from_bits(self.tcx, 0, self.typing_env(), ty); self.literal_operand(span, literal) } diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs index 39bd2be9409a3..bb0685cb7ff08 100644 --- a/compiler/rustc_mir_dataflow/src/framework/direction.rs +++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs @@ -127,6 +127,8 @@ impl Direction for Backward { analysis.apply_primary_statement_effect(state, stmt, loc); vis.visit_after_primary_statement_effect(state, stmt, loc); } + + vis.visit_block_exit(state, block); } } @@ -242,5 +244,7 @@ impl Direction for Forward { vis.visit_after_early_terminator_effect(state, term, loc); analysis.apply_primary_terminator_effect(state, term, loc); vis.visit_after_primary_terminator_effect(state, term, loc); + + vis.visit_block_exit(state, block); } } diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs index e4b840a73e502..5b8a3374e04d9 100644 --- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs +++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs @@ -34,6 +34,13 @@ pub trait ResultsVisitor<'tcx, A> where A: Analysis<'tcx>, { + /// Called after all effects in a block have been applied in the direction + /// of the analysis. + /// + /// In a forwards analysis, `state` is from the block's end. In a backwards + /// analysis, `state` is from the block's start. + fn visit_block_exit(&mut self, _state: &A::Domain, _block: BasicBlock) {} + /// Called after the "early" effect of the given statement is applied to `state`. fn visit_after_early_statement_effect( &mut self, diff --git a/compiler/rustc_mir_dataflow/src/impls/mod.rs b/compiler/rustc_mir_dataflow/src/impls/mod.rs index 1e12e41ce1fb4..faa4b0dd1694b 100644 --- a/compiler/rustc_mir_dataflow/src/impls/mod.rs +++ b/compiler/rustc_mir_dataflow/src/impls/mod.rs @@ -1,6 +1,7 @@ mod borrowed_locals; mod initialized; mod liveness; +mod precise_liveness; mod storage_liveness; pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals}; @@ -11,6 +12,10 @@ pub use self::initialized::{ pub use self::liveness::{ DefUse, LivenessTransferFunction, MaybeLiveLocals, MaybeTransitiveLiveLocals, }; +pub use self::precise_liveness::{ + SplitPointEffect, SplitPointIndex, VisitAllocatedLocalsWith, VisitDeallocatedLocalsWith, + dump_liveness_matrix, liveness_matrix, +}; pub use self::storage_liveness::{ MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals, }; diff --git a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs new file mode 100644 index 0000000000000..0234c240ef304 --- /dev/null +++ b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs @@ -0,0 +1,596 @@ +//! Computes the points where each local must have a distinct allocation. +//! +//! The result is a [`SparseIntervalMatrix`] with one row per local. Two locals +//! may share the same address only if their rows are disjoint. To model MIR +//! statements where a source operand and destination place may share an +//! address, each statement and terminator is split into an early point, where +//! operands are read, and a late point, where destinations are written. +//! +//! A local live range starts at the late point of any statement or terminator +//! that writes to it without a `Deref` projection. It ends at `StorageLive` or +//! `StorageDead`, or at the early point of a whole-local move operand or the +//! last use of that local on a control-flow path (only for locals whose address +//! is never observed). This happens at `StorageLive` because it resets a local +//! to an uninitialized state. +//! +//! `Call` terminators are handled specially: move operands are kept live +//! through the late point of the terminator so they conflict with each other +//! and with the destination place. This matches the runtime behavior where the +//! place is donated to the callee for the duration of the call. +//! +//! The analysis consists of three passes: +//! +//! 1. A backward value-liveness analysis which identifies the last point at +//! which each local is directly used in each control flow path. This is used +//! to shorten the lifetime of unborrowed locals to their last use. +//! 2. A forward allocation-liveness analysis which uses that information +//! together with writes, moves, and storage markers to compute which +//! allocations may be live on entry to each block. +//! 3. A final walk using those entry states which constructs the matrix and +//! distinguishes whether each effect happens at the early or late point of a +//! statement or terminator. + +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{ + MutatingUseContext, NonMutatingUseContext, NonUseContext, PlaceContext, VisitPlacesWith, + Visitor, +}; +use rustc_middle::mir::{self, BasicBlock, Local, Location, MirDumper, PassWhere, Place}; +use rustc_middle::ty::TyCtxt; +use tracing::trace; + +use crate::impls::{DefUse, MaybeLiveLocals, borrowed_locals}; +use crate::points::{DenseLocationMap, PointIndex}; +use crate::{Analysis, GenKill, ResultsVisitor, visit_results}; + +/// Visits locals that are implicitly allocated by a statement or terminator as +/// part of destination place evaluation. +pub struct VisitAllocatedLocalsWith(pub F); + +impl<'tcx, F> Visitor<'tcx> for VisitAllocatedLocalsWith +where + F: FnMut(Local, PlaceContext), +{ + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _: Location) { + match DefUse::for_place(*place, context) { + DefUse::Def | DefUse::PartialWrite => (self.0)(place.local, context), + DefUse::Use | DefUse::NonUse => {} + } + } +} + +/// Visits locals that are implicitly freed by being evaluated as a bare-local +/// move operand or through a storage statement. +pub struct VisitDeallocatedLocalsWith(pub F); + +impl<'tcx, F> Visitor<'tcx> for VisitDeallocatedLocalsWith +where + F: FnMut(Local, PlaceContext), +{ + fn visit_statement(&mut self, statement: &mir::Statement<'tcx>, location: Location) { + match statement.kind { + // StorageLive resets a local to the live-but-unallocated state, + // which also has the effect of freeing its allocation. + mir::StatementKind::StorageLive(local) => { + (self.0)(local, PlaceContext::NonUse(NonUseContext::StorageLive)); + } + mir::StatementKind::StorageDead(local) => { + (self.0)(local, PlaceContext::NonUse(NonUseContext::StorageDead)); + } + _ => self.super_statement(statement, location), + } + } + + fn visit_operand(&mut self, operand: &mir::Operand<'tcx>, _: Location) { + if let mir::Operand::Move(place) = operand + && let Some(local) = place.as_local() + { + (self.0)(local, PlaceContext::NonMutatingUse(NonMutatingUseContext::Move)); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Backward dataflow pass +// +// This pass computes "kill points" for each local, indicating the location of +// their last use in a particular control flow branch. These are later used in +// the forward pass to end the live range of locals that are never borrowed at +// their last direct use. +// +// This pass ignores borrowed locals: we can't determine their last use from +// syntactic uses of the base local, so we conservatively treat them as always +// live in the backwards pass and don't emit kill points for them. The main +// forward pass still kills at `StorageDead` and whole-local moves, which is +// when their allocation is freed. +// +// This pass has 2 outputs: a set of kill points that mark the last use +// locations of locals and a per-block bitset indicating which locals are live +// on entry to that block. The latter acts as a set of kill points at block +// entry for all locals not in the bit set. + +struct KillPoints<'a> { + live_on_entry: IndexVec>, + kill_points_map: IndexVec, +} + +impl<'a> KillPoints<'a> { + fn compute<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: Option<&'static str>, + points: &DenseLocationMap, + kill_points: &'a mut Vec<(Local, Location)>, + ) -> Self { + let maybe_live_locals = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, pass_name); + let borrowed_locals = borrowed_locals(body); + + // Initialize all borrowed locals as live on entry. We never try to kill + // those. + let mut live_on_entry = + IndexVec::from_elem_n(borrowed_locals.clone(), body.basic_blocks.len()); + + // Collect kill points and live-on-entry states from the results of + // MaybeLiveLocals. + kill_points.clear(); + let mut visitor = KillPointsVisitor { + kill_points, + live_on_entry: &mut live_on_entry, + borrowed_locals: &borrowed_locals, + }; + visit_results( + body, + mir::traversal::reachable(body).map(|(block, _)| block), + &maybe_live_locals, + &mut visitor, + ); + trace!(?kill_points); + trace!(?live_on_entry); + + // Create a mapping of `PointIndex` to the set of killed locals at that + // location. + let mut kill_points_map = IndexVec::from_elem_n(&[][..], points.num_points()); + for chunk in kill_points.chunk_by(|a, b| a.1 == b.1) { + let point = points.point_from_location(chunk[0].1); + trace!("Kill points at {:?}: {:?}", chunk[0].1, chunk); + debug_assert!(kill_points_map[point].is_empty()); + kill_points_map[point] = chunk; + } + + Self { live_on_entry, kill_points_map } + } +} + +struct KillPointsVisitor<'a> { + kill_points: &'a mut Vec<(Local, Location)>, + live_on_entry: &'a mut IndexVec>, + borrowed_locals: &'a DenseBitSet, +} + +impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { + fn visit_block_exit(&mut self, state: &DenseBitSet, block: BasicBlock) { + // Borrowed locals are already marked as live when live_on_entry was + // initialized. This adds the non-borrowed locals that we have + // determined are live on entry to this block. + self.live_on_entry[block].union(state); + } + + fn visit_after_early_statement_effect( + &mut self, + state: &DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses. + match ctxt { + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + PlaceContext::NonUse(_) => return, + } + + // If a local is used in a statement but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_statement(statement, location); + } + + fn visit_after_early_terminator_effect( + &mut self, + state: &DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses (they don't do anything) and edge uses + // (implicitly killed though live_on_entry at the start of the + // corresponding successor). + match ctxt { + PlaceContext::MutatingUse( + MutatingUseContext::AsmOutput + | MutatingUseContext::Call + | MutatingUseContext::Yield, + ) + | PlaceContext::NonUse(_) => return, + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + } + + // If a local is used in a terminator but is dead after it then this + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_terminator(terminator, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Forward dataflow pass +// +// This is a simpler version of the main pass in `liveness_matrix`. It only +// needs to determine local liveness at block edges and doesn't distinguish +// between "early" and "late" effects. However it must still preserve the +// ordering of effects: moved operands are killed before destinations are +// generated. + +struct PreciseLiveness<'a> { + kill_points: &'a KillPoints<'a>, + points: &'a DenseLocationMap, +} + +impl PreciseLiveness<'_> { + fn apply_block_start_effect(&self, state: &mut DenseBitSet, block: BasicBlock) { + // Kill any unborrowed locals that are dead at the start of this block. + // Notably this kills any dead results produced by a predecessor's + // terminator. Borrowed locals are always included in live_on_entry. + state.intersect(&self.kill_points.live_on_entry[block]); + } +} + +impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { + type Domain = DenseBitSet; + + const NAME: &'static str = "precise_liveness"; + + fn bottom_value(&self, body: &mir::Body<'tcx>) -> DenseBitSet { + DenseBitSet::new_empty(body.local_decls.len()) + } + + fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut DenseBitSet) { + // Function arguments start out as live. + for arg in body.args_iter() { + state.gen_(arg); + } + } + + fn apply_primary_statement_effect( + &self, + state: &mut DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + VisitDeallocatedLocalsWith(|local, _| state.kill(local)) + .visit_statement(statement, location); + VisitAllocatedLocalsWith(|local, _| state.gen_(local)).visit_statement(statement, location); + + // Apply kill points at this statement: if a variable is dead then it + // doesn't need storage. + let point = self.points.point_from_location(location); + for &(local, _) in self.kill_points.kill_points_map[point] { + state.kill(local); + } + } + + fn apply_primary_terminator_effect( + &self, + state: &mut DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + VisitDeallocatedLocalsWith(|local, _| state.kill(local)) + .visit_terminator(terminator, location); + + // Gen destination places. + VisitAllocatedLocalsWith(|local, ctxt| { + // The resume place of a Yield is only evaluated after the coroutine + // is resumed. This is in contrast to call destinations and inline + // asm outputs which are evaluated before the call and therefore + // remain allocated on both normal and unwind edges. + if ctxt != PlaceContext::MutatingUse(MutatingUseContext::Yield) { + state.gen_(local); + } + }) + .visit_terminator(terminator, location); + } + + fn apply_call_return_effect( + &self, + state: &mut DenseBitSet, + _block: BasicBlock, + return_places: mir::CallReturnPlaces<'_, 'tcx>, + ) { + if let mir::CallReturnPlaces::Yield(place) = return_places + && !place.is_indirect() + { + state.gen_(place.local); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Matrix construction + +/// Different "phases" of a single MIR statement, used to describe how +/// overlapping operands are handled. +/// +/// As a general rule, source operands are read in the `Early` phase and +/// destination places are written in the `Late` phase. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum SplitPointEffect { + Early = 0, + Late = 1, +} + +rustc_index::newtype_index! { + /// A `PointIndex` with the lower bit encoding early/late inside a + /// statement. + /// + /// This is used to model overlap constraints within a MIR statement: if a + /// source/destination are allowed to overlap then the source is read in + /// `SplitPointEffect::Early` and the write is done in + /// `SplitPointEffect::Late`. + #[orderable] + #[debug_format = "SplitPointIndex({})"] + pub struct SplitPointIndex {} +} + +impl SplitPointIndex { + pub fn new(point: PointIndex, effect: SplitPointEffect) -> SplitPointIndex { + let index = (point.as_u32() << 1) | (effect as u32); + SplitPointIndex::from_u32(index) + } + + pub fn point(self) -> PointIndex { + PointIndex::from_u32(self.as_u32() >> 1) + } + + pub fn effect(self) -> SplitPointEffect { + match self.as_u32() & 1 { + 0 => SplitPointEffect::Early, + 1 => SplitPointEffect::Late, + _ => unreachable!(), + } + } +} + +/// Helper type to construct a `SparseIntervalMatrix`. +struct MatrixBuilder { + matrix: SparseIntervalMatrix, + range_start: IndexVec>, + + // Track locals that have been live at any point in a block so that at the + // end of a block we don't need to iterate over all locals. This + // significantly speeds up matrix building. + maybe_live_locals: Vec, +} + +impl MatrixBuilder { + fn gen_(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let split_point = SplitPointIndex::new(point, effect); + + // No-op if the local is already live. + if self.range_start[local].is_none() { + self.range_start[local] = Some(split_point); + self.maybe_live_locals.push(local); + } + } + + fn kill(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let end = SplitPointIndex::new(point, effect); + + // No-op if the local is already dead. + if let Some(start) = self.range_start[local].take() { + debug_assert!(end >= start); + self.matrix.append_range(local, start..=end); + } + } + + fn kill_all(&mut self, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + self.kill(local, point, effect); + } + } + + fn kill_all_except(&mut self, except: Local, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + if local != except { + self.kill(local, point, effect); + } + } + self.maybe_live_locals.push(except); + } +} + +pub fn liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + points: &DenseLocationMap, + pass_name: Option<&'static str>, +) -> SparseIntervalMatrix { + let mut kill_points_vec = vec![]; + let kill_points = KillPoints::compute(tcx, body, pass_name, points, &mut kill_points_vec); + let mut results = PreciseLiveness { kill_points: &kill_points, points } + .iterate_to_fixpoint(tcx, body, pass_name); + + let mut builder = MatrixBuilder { + matrix: SparseIntervalMatrix::new(points.num_points() * 2), + range_start: IndexVec::from_elem_n(None, body.local_decls.len()), + maybe_live_locals: Vec::new(), + }; + for (block, block_data) in body.basic_blocks.iter_enumerated() { + // We can mutate the state in-place since we're not using it any more + // after this point. + let state = &mut results.entry_states[block]; + + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.intersect(&kill_points.live_on_entry[block]); + + // Gen any locals that are live at the start of the block. If this block + // only consists of a return terminator then only gen the return place + // instead. This ensures that StorageDead for all other locals are + // inserted before the return terminator. + let terminator = block_data.terminator(); + if let mir::TerminatorKind::Return = terminator.kind + && block_data.statements.is_empty() + { + if state.contains(mir::RETURN_PLACE) { + builder.gen_(mir::RETURN_PLACE, points.entry_point(block), SplitPointEffect::Early); + } + } else { + for local in state.iter() { + builder.gen_(local, points.entry_point(block), SplitPointEffect::Early); + } + } + + for (statement_index, statement) in block_data.statements.iter().enumerate() { + let location = Location { block, statement_index }; + let point = points.point_from_location(location); + + // Kill moved locals at the early point. + VisitDeallocatedLocalsWith(|local, _| { + builder.kill(local, point, SplitPointEffect::Early); + }) + .visit_statement(statement, location); + + // Kill any locals which are no longer used after this statement. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places at the late point. + VisitAllocatedLocalsWith(|local, _| { + builder.gen_(local, point, SplitPointEffect::Late); + }) + .visit_statement(statement, location); + + // Kill any dead destination places: they will only appear at the + // late point of the statement they are generated in, which is + // sufficient for determining overlap. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Late); + } + } + + // If this block ends in a return terminator, end all live ranges before + // the terminator so that StorageDead statements are inserted before it. + // + // This is useful after inlining so that the lifetime of locals in the + // inlined callee don't extend past the call in the caller. + if let mir::TerminatorKind::Return = terminator.kind + && !block_data.statements.is_empty() + { + // Blocks with only a return terminator are handled above. + let location = Location { block, statement_index: block_data.statements.len() - 1 }; + let point = points.point_from_location(location); + builder.kill_all_except(mir::RETURN_PLACE, point, SplitPointEffect::Late); + } + + let location = Location { block, statement_index: block_data.statements.len() }; + let point = points.point_from_location(location); + + // Kill moved locals at the early point. + VisitDeallocatedLocalsWith(|local, _| { + builder.kill(local, point, SplitPointEffect::Early); + }) + .visit_terminator(terminator, location); + + // Kill any locals which are no longer used after this terminator. + for &(local, _) in kill_points.kill_points_map[point] { + builder.kill(local, point, SplitPointEffect::Early); + } + + // Gen destination places at the late point. + VisitAllocatedLocalsWith(|local, _| { + builder.gen_(local, point, SplitPointEffect::Late); + }) + .visit_terminator(terminator, location); + + // Move arguments to a call are treated specially: the place that they + // represent is passed directly to the callee, which means that they are + // not allowed to alias any other move operand or the destination place. + // This is represented here by extending their live range to the late + // part, making it overlap with that of the destination place. + // + // Notably, this *doesn't* apply to TailCall. + if let mir::TerminatorKind::Call { + func: _, + args, + destination: _, + target: _, + unwind: _, + call_source: _, + fn_span: _, + } = &terminator.kind + { + for arg in args { + if let mir::Operand::Move(place) = arg.node + && !place.is_indirect() + { + builder.gen_(place.local, point, SplitPointEffect::Late); + } + } + } + + // End the lifetimes of all locals at the end of the block. Successor + // blocks (which may not be contiguous in the index space!) will + // initialize the lifetimes again from their entry state. + builder.kill_all(point, SplitPointEffect::Late); + } + + builder.matrix +} + +pub fn dump_liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: &'static str, + points: &DenseLocationMap, + matrix: &SparseIntervalMatrix, +) { + let locals_live_at = |split_point| { + matrix.rows().filter(|&r| matrix.contains(r, split_point)).collect::>() + }; + + if let Some(dumper) = MirDumper::new(tcx, pass_name, body) { + let extra_data = &|pass_where, w: &mut dyn std::io::Write| { + if let PassWhere::BeforeLocation(loc) = pass_where { + let point = points.point_from_location(loc); + let split_point = SplitPointIndex::new(point, SplitPointEffect::Early); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-early => {live:?}")?; + let split_point = SplitPointIndex::new(point, SplitPointEffect::Late); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-late => {live:?}")?; + } + Ok(()) + }; + + dumper.set_extra_data(extra_data).dump_mir(body) + } +} diff --git a/compiler/rustc_mir_dataflow/src/pretty.rs b/compiler/rustc_mir_dataflow/src/pretty.rs index b37ad708e20e0..5a159ae40bd3c 100644 --- a/compiler/rustc_mir_dataflow/src/pretty.rs +++ b/compiler/rustc_mir_dataflow/src/pretty.rs @@ -7,7 +7,11 @@ use rustc_middle::ty::TyCtxt; use crate::debuginfo::debuginfo_locals; use crate::framework::Analysis; -use crate::impls::{MaybeLiveLocals, MaybeTransitiveLiveLocals, borrowed_locals}; +use crate::impls::{ + MaybeLiveLocals, MaybeTransitiveLiveLocals, SplitPointEffect, SplitPointIndex, borrowed_locals, + liveness_matrix, +}; +use crate::points::DenseLocationMap; use crate::{ResultsVisitor, visit_results}; type ExtraDataFn = dyn Fn(PassWhere, &mut dyn io::Write) -> io::Result<()>; @@ -33,6 +37,29 @@ pub(crate) fn mir_pretty_extra_data<'tcx>( &mut Annotator { annotations: &mut annotations }, ); } + RustcMirKind::PrettyPreciseLiveness => { + let points = DenseLocationMap::new(body); + let matrix = liveness_matrix(tcx, body, &points, None); + for (block, data) in body.basic_blocks.iter_enumerated() { + for statement_index in 0..=data.statements.len() { + let location = mir::Location { block, statement_index }; + let point = points.point_from_location(location); + let locals_live_at = |effect| { + let split_point = SplitPointIndex::new(point, effect); + matrix + .rows() + .filter(|&r| matrix.contains(r, split_point)) + .collect::>() + }; + let early = locals_live_at(SplitPointEffect::Early); + annotations + .add(PassWhere::BeforeLocation(location), format!("early: {early:?}")); + let late = locals_live_at(SplitPointEffect::Late); + annotations + .add(PassWhere::BeforeLocation(location), format!("late: {late:?}")); + } + } + } RustcMirKind::PrettyTransitiveLiveLocals => { let borrowed_locals = borrowed_locals(body); let debuginfo_locals = debuginfo_locals(body); diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs index 655ab64145db5..00dfc146db50a 100644 --- a/compiler/rustc_next_trait_solver/src/canonical/mod.rs +++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs @@ -33,17 +33,17 @@ use crate::solve::{ pub mod canonicalizer; trait ResponseT { - fn var_values(&self) -> CanonicalVarValues; + fn var_values(&self) -> I::GenericArgs; } impl ResponseT for Response { - fn var_values(&self) -> CanonicalVarValues { + fn var_values(&self) -> I::GenericArgs { self.var_values } } impl ResponseT for inspect::State { - fn var_values(&self) -> CanonicalVarValues { + fn var_values(&self) -> I::GenericArgs { self.var_values } } @@ -210,7 +210,7 @@ where // We therefore instantiate the existential variable in the canonical response with the // inference variable of the input right away, which is more performant. let mut opt_values = IndexVec::from_elem_n(None, response.var_kinds.len()); - for (original_value, result_value) in iter::zip(original_values, var_values.var_values.iter()) { + for (original_value, result_value) in iter::zip(original_values, var_values.iter()) { match result_value.kind() { ty::GenericArgKind::Type(t) => { // We disable the instantiation guess for inference variables @@ -490,7 +490,7 @@ where fn unify_query_var_values( delegate: &D, original_values: &[I::GenericArg], - var_values: CanonicalVarValues, + var_values: I::GenericArgs, span: I::Span, ) where D: SolverDelegate, @@ -498,7 +498,7 @@ fn unify_query_var_values( { assert_eq!(original_values.len(), var_values.len()); - for (&orig, response) in iter::zip(original_values, var_values.var_values.iter()) { + for (&orig, response) in iter::zip(original_values, var_values.iter()) { let mut must_eq = ResponseRelating::new(&**delegate, span); must_eq.relate(orig, response).unwrap(); } @@ -564,7 +564,7 @@ where I: Interner, T: TypeFoldable, { - let var_values = CanonicalVarValues { var_values: delegate.cx().mk_args(var_values) }; + let var_values = delegate.cx().mk_args(var_values); let state = inspect::State { var_values, data }; let state = delegate.deeply_resolve_via_unification_table(state); Canonicalizer::canonicalize_response(delegate, max_input_universe, state) @@ -591,7 +591,7 @@ where delegate.create_next_universe(); } orig_values.extend( - state.value.var_values.var_values.as_slice()[orig_values.len()..] + state.value.var_values.as_slice()[orig_values.len()..] .iter() .map(|&arg| delegate.fresh_var_for_kind(arg, span, max_universe)), ); @@ -616,11 +616,31 @@ pub fn response_no_constraints_raw( var_kinds: I::CanonicalVarKinds, certainty: Certainty, ) -> CanonicalResponse { + // Given a list of canonical variables, construct a set of values which are + // the identity response. + let var_values = + cx.mk_args_from_iter(var_kinds.iter().enumerate().map(|(i, kind)| -> I::GenericArg { + match kind { + CanonicalVarKind::Ty { .. } + | CanonicalVarKind::Int + | CanonicalVarKind::Float + | CanonicalVarKind::PlaceholderTy(_) => { + Ty::new_canonical_bound(cx, ty::BoundVar::from_usize(i)).into() + } + CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => { + Region::new_canonical_bound(cx, ty::BoundVar::from_usize(i)).into() + } + CanonicalVarKind::Const(_) | CanonicalVarKind::PlaceholderConst(_) => { + Const::new_canonical_bound(cx, ty::BoundVar::from_usize(i)).into() + } + } + })); + ty::Canonical { max_universe, var_kinds, value: Response { - var_values: ty::CanonicalVarValues::make_identity(cx, var_kinds), + var_values, // FIXME: maybe we should store the "no response" version in cx, like // we do for cx.types and stuff. external_constraints: cx.mk_external_constraints(ExternalConstraintsData::new(cx)), diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 044dc89978124..f97bc0bbeb626 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -17,9 +17,9 @@ use rustc_type_ir::solve::{ RerunNonErased, RerunReason, RerunResultExt, SmallCopySet, TyOrConstInferVar, }; use rustc_type_ir::{ - self as ty, CanonicalVarValues, ClauseKind, Const, InferCtxtLike, Interner, MayBeErased, - OpaqueTypeKey, PredicateKind, PredicateProxy, Region, RegionVid, TypeFoldable, - TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor, TypingMode, max_universe, + self as ty, ClauseKind, Const, InferCtxtLike, Interner, MayBeErased, OpaqueTypeKey, + PredicateKind, PredicateProxy, Region, RegionVid, TypeFoldable, TypeSuperVisitable, + TypeVisitable, TypeVisitableExt, TypeVisitor, TypingMode, max_universe, }; use thin_vec::ThinVec; use tracing::{Level, debug, instrument, trace, warn}; @@ -129,7 +129,7 @@ where /// What kind of goal we're currently computing, see the enum definition /// for more info. current_goal_kind: CurrentGoalKind, - pub(super) var_values: CanonicalVarValues, + pub(super) var_values: I::GenericArgs, /// The highest universe index nameable by the caller. /// @@ -479,7 +479,7 @@ where max_input_universe: ty::UniverseIndex::ROOT, initial_opaque_types_storage_num_entries: Default::default(), var_kinds: Default::default(), - var_values: CanonicalVarValues::dummy(), + var_values: Default::default(), current_goal_kind: CurrentGoalKind::Misc, origin_span, tainted: Ok(()), @@ -513,6 +513,7 @@ where ) -> Result, ) -> (Result, AccessedOpaques) { let (ref delegate, input, var_values) = D::build_with_canonical(cx, &canonical_input); + let var_values = var_values.var_values; for (key, ty) in input.predefined_opaques_in_body.iter() { let prev = delegate.register_hidden_type_in_storage(key, ty, I::Span::dummy()); // It may be possible that two entries in the opaque type storage end up @@ -1707,7 +1708,7 @@ where r.retain(|(outlives, _)| !outlives.is_trivial() && unique.insert(*outlives)); } - filter_irrelevant_region_constraints(self.delegate, &var_values, &mut external_constraints); + filter_irrelevant_region_constraints(self.delegate, var_values, &mut external_constraints); let canonical = canonicalize_response( self.delegate, @@ -1826,7 +1827,7 @@ where fn filter_irrelevant_region_constraints( delegate: &D, - var_values: &CanonicalVarValues, + var_values: I::GenericArgs, external_constraints: &mut ExternalConstraintsData, ) where D: SolverDelegate, diff --git a/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs b/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs index 4369148baf91d..9584e16d68725 100644 --- a/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs +++ b/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs @@ -46,14 +46,14 @@ impl, I: Interner> ProofTreeBuilder { pub(crate) fn new_evaluation_step( &mut self, - var_values: ty::CanonicalVarValues, + var_values: I::GenericArgs, ) -> EvaluationStepBuilder { if self.is_noop() { EvaluationStepBuilder { state: None, _infcx: PhantomData } } else { EvaluationStepBuilder { state: Some(Box::new(WipEvaluationStep { - var_values: var_values.var_values.to_vec(), + var_values: var_values.to_vec(), evaluation: WipProbe { initial_num_var_values: var_values.len(), steps: vec![], diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index 099c755d78427..9ed4963577458 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -62,7 +62,7 @@ fn has_no_inference_or_external_constraints( ref opaque_types, ref normalization_nested_goals, } = *response.value.external_constraints; - response.value.var_values.is_identity() + response.value.is_identity() && region_constraints.is_empty() && opaque_types.is_empty() && normalization_nested_goals.is_empty() @@ -74,7 +74,7 @@ fn has_only_region_constraints(response: ty::Canonical