Remap def indices for deterministic metadata encoding

compiler/rustc_hir_id/src/definitions.rs

@@ -6,6 +6,7 @@
 
 use std::fmt::{self, Write};
 use std::hash::Hash;
+use std::sync::OnceLock;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
@@ -95,6 +96,7 @@ pub struct Definitions {
     def_id_to_key: IndexVec<LocalDefId, DefKey>,
     // We do only store the local hash, as all the definitions are from the current crate.
     def_path_hashes: IndexVec<LocalDefId, Hash64>,
+    last_deterministic_index: OnceLock<DefIndex>,
     def_path_hash_to_index: DefPathToIndexMap,
 }
 
@@ -292,13 +294,21 @@ pub enum DefPathData {
 }
 
 impl Definitions {
-    /// This function indicates that the order of def id allocations
-    /// may be non-deterministic after it was called.
     pub fn commit_end_of_determinism(&mut self) {
         assert!(
             self.def_path_hash_to_index.after_parallel_alloc.replace(Default::default()).is_none(),
             "this function should be called only once"
-        )
+        );
+
+        self.last_deterministic_index
+            .set(
+                if self.def_id_to_key.is_empty() { 0 } else { self.def_id_to_key.len() - 1 }.into(),
+            )
+            .expect("must be called once");
+    }
+
+    pub fn last_deterministic_index(&self) -> DefIndex {
+        self.last_deterministic_index.get().copied().expect("must contain index")
     }
 
     #[inline(always)]
@@ -352,6 +362,7 @@ impl Definitions {
             def_path_hashes: Default::default(),
             def_id_to_key: Default::default(),
             def_path_hash_to_index: Default::default(),
+            last_deterministic_index: Default::default(),
         };
 
         // Create the root definition.

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -1967,8 +1967,12 @@ impl CrateMetadata {
         let trait_impls = root
             .impls
             .decode(&blob)
-            .map(|trait_impls| (trait_impls.trait_id, trait_impls.impls))
+            .map(|trait_impls| {
+                let (krate, index) = trait_impls.trait_id;
+                ((krate, DefIndex::from_u32(index)), trait_impls.impls)
+            })
             .collect();
+
         let alloc_decoding_state =
             AllocDecodingState::new(root.interpret_alloc_index.decode(&blob).collect());
 

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -21,7 +21,7 @@ use rustc_hir as hir;
 use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId, LocalDefIdSet};
 use rustc_hir::definitions::DefPathData;
 use rustc_hir_pretty::id_to_string;
-use rustc_index::IndexVec;
+use rustc_index::{Idx, IndexVec};
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::hir::map::compute_hir_hash;
 use rustc_middle::middle::dependency_format::Linkage;
@@ -78,6 +78,8 @@ pub(super) struct EncodeContext<'a, 'tcx> {
     hygiene_ctxt: Rc<RefCell<HygieneEncodeContext>>,
     // Used for both `Symbol`s and `ByteSymbol`s.
     symbol_index_table: FxHashMap<u32, usize>,
+    pub(super) def_indexes_remapping: FxHashMap<DefIndex, DefIndex>,
+    last_deterministic_index: u32,
 }
 
 /// If the current crate is a proc-macro, returns early with `LazyArray::default()`.
@@ -156,12 +158,14 @@ impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> {
     }
 
     fn encode_def_index(&mut self, def_index: DefIndex) {
-        self.emit_u32(def_index.as_u32());
+        self.emit_u32(self.map_index(def_index).as_u32());
     }
 
     fn encode_def_id(&mut self, def_id: DefId) {
+        let def_id = self.map_def_id(def_id);
+
         def_id.krate.encode(self);
-        def_id.index.encode(self);
+        self.emit_u32(def_id.index.as_u32());
     }
 
     fn encode_syntax_context(&mut self, syntax_context: SyntaxContext) {
@@ -409,20 +413,23 @@ macro_rules! record_some_lazy {
     ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
         let value = $value;
         let lazy = $self.lazy(value);
-        $self.$tables.$table.set_some($def_id.index, lazy);
+        let def_id = $self.map_def_id($def_id);
+        $self.$tables.$table.set_some(def_id.index, lazy);
     }};
 }
 
 macro_rules! record_some {
-    ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {
-        $self.$tables.$table.set_some($def_id.index, $value)
-    };
+    ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
+        let def_id = $self.map_def_id($def_id);
+        $self.$tables.$table.set_some(def_id.index, $value)
+    }};
 }
 
 macro_rules! record_value {
-    ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {
-        $self.$tables.$table.set($def_id.index, $value)
-    };
+    ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
+        let def_id = $self.map_def_id($def_id);
+        $self.$tables.$table.set(def_id.index, $value)
+    }};
 }
 
 // Shorthand for `$self.$tables.$table.set_some($def_id.index, $self.lazy_array($value))`, which would
@@ -431,15 +438,17 @@ macro_rules! record_array {
     ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
         let value = $value;
         let lazy = $self.lazy_array(value);
-        $self.$tables.$table.set_some($def_id.index, lazy);
+        let def_id = $self.map_def_id($def_id);
+        $self.$tables.$table.set_some(def_id.index, lazy);
     }};
 }
 
 macro_rules! record_defaulted_array {
     ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
         let value = $value;
         let lazy = $self.lazy_array(value);
-        $self.$tables.$table.set($def_id.index, lazy);
+        let def_id = $self.map_def_id($def_id);
+        $self.$tables.$table.set(def_id.index, lazy);
     }};
 }
 
@@ -528,7 +537,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         }
     }
 
-    fn encode_def_path_table(&mut self) {
+    fn encode_def_path_table(&mut self, sorted_ids: &[LocalDefId]) {
         let defs = self.tcx.definitions();
         if self.is_proc_macro {
             for def_id in std::iter::once(CRATE_DEF_ID)
@@ -542,10 +551,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record_value!(self.tables.def_path_hashes[def_id] <- def_path_hash.local_hash().as_u64())
             }
         } else {
-            for (def_index, def_key, def_path_hash) in defs.enumerated_keys_and_path_hashes() {
-                let def_id = LocalDefId { local_def_index: def_index }.to_def_id();
+            for &def_id in sorted_ids {
+                let def_key = defs.def_key(def_id);
+                let hash = defs.def_path_hash(def_id).local_hash().as_u64();
+
+                let def_id = def_id.to_def_id();
+
                 record_some_lazy!(self.tables.def_keys[def_id] <- def_key);
-                record_value!(self.tables.def_path_hashes[def_id] <- def_path_hash.local_hash().as_u64())
+                record_value!(self.tables.def_path_hashes[def_id] <- hash);
             }
         }
     }
@@ -624,7 +637,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         adapted.encode(&mut self.opaque)
     }
 
-    fn encode_crate_root(&mut self) -> (LazyValue<CrateRoot>, LazyValue<CrateRootUnhashed>) {
+    fn encode_crate_root(
+        &mut self,
+        remapping: FxHashMap<DefIndex, DefIndex>,
+        sorted_ids: Vec<LocalDefId>,
+    ) -> (LazyValue<CrateRoot>, LazyValue<CrateRootUnhashed>) {
+        self.def_indexes_remapping = remapping;
+
         let tcx = self.tcx;
         let mut stats: Vec<(&'static str, usize)> = Vec::with_capacity(32);
 
@@ -667,7 +686,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         let foreign_modules = stat!("foreign-modules", || self.encode_foreign_modules());
 
-        _ = stat!("def-path-table", || self.encode_def_path_table());
+        _ = stat!("def-path-table", || self.encode_def_path_table(&sorted_ids));
 
         // Encode the def IDs of traits, for rustdoc and diagnostics.
         let traits = stat!("traits", || self.encode_traits());
@@ -679,7 +698,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         _ = stat!("mir", || self.encode_mir());
 
-        _ = stat!("def-ids", || self.encode_def_ids());
+        _ = stat!("def-ids", || self.encode_def_ids(&sorted_ids));
 
         let interpret_alloc_index = stat!("interpret-alloc-index", || {
             let mut interpret_alloc_index = Vec::new();
@@ -1455,7 +1474,26 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         record_value!(self.tables.attr_flags[def_id.to_def_id()] <- attr_flags)
     }
 
-    fn encode_def_ids(&mut self) {
+    fn map_def_id(&self, def_id: DefId) -> DefId {
+        if def_id.is_local() {
+            DefId { krate: LOCAL_CRATE, index: self.map_index(def_id.index) }
+        } else {
+            def_id
+        }
+    }
+
+    #[inline]
+    fn map_index(&self, def_index: DefIndex) -> DefIndex {
+        let index = def_index.as_u32();
+
+        if index <= self.last_deterministic_index {
+            def_index
+        } else {
+            self.def_indexes_remapping.get(&def_index).copied().unwrap_or(def_index)
+        }
+    }
+
+    fn encode_def_ids(&mut self, sorted_ids: &[LocalDefId]) {
         self.encode_info_for_mod(CRATE_DEF_ID);
 
         // Proc-macro crates only export proc-macro items, which are looked
@@ -1465,8 +1503,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         }
 
         let tcx = self.tcx;
-
-        for local_id in tcx.iter_local_def_id() {
+        for &local_id in sorted_ids {
             let def_id = local_id.to_def_id();
             let def_kind = tcx.def_kind(local_id);
             record_some!(self.tables.def_kind[def_id] <- def_kind);
@@ -2302,7 +2339,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 if let Ok(mut an) = trait_def.ancestors(tcx, def_id)
                     && let Some(specialization_graph::Node::Impl(parent)) = an.nth(1)
                 {
-                    record_some!(self.tables.impl_parent[def_id] <- parent.into());
+                    record_some!(self.tables.impl_parent[def_id] <- self.map_def_id(parent).into());
                 }
 
                 // if this is an impl of `CoerceUnsized`, create its
@@ -2316,9 +2353,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         let trait_impls: Vec<_> = trait_impls
             .into_iter()
-            .map(|(trait_def_id, impls)| TraitImpls {
-                trait_id: (trait_def_id.krate.as_u32(), trait_def_id.index),
-                impls: self.lazy_array(&impls),
+            .map(|(trait_id, impls)| {
+                let trait_id = self.map_def_id(trait_id);
+                TraitImpls {
+                    trait_id: (trait_id.krate.as_u32(), trait_id.index.as_u32()),
+                    impls: self.lazy_array(&impls),
+                }
             })
             .collect();
 
@@ -2601,7 +2641,9 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) {
             with_encode_metadata_header(tcx, path, |ecx| {
                 // Encode all the entries and extra information in the crate,
                 // culminating in the `CrateRoot` which points to all of it.
-                let (root, unhashed) = ecx.encode_crate_root();
+                let (remapping, sequence) = create_def_index_remapping(tcx);
+
+                let (root, unhashed) = ecx.encode_crate_root(remapping, sequence);
 
                 // Flush buffer to ensure backing file has the correct size.
                 ecx.opaque.flush();
@@ -2634,6 +2676,40 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) {
     }
 }
 
+fn create_def_index_remapping(tcx: TyCtxt<'_>) -> (FxHashMap<DefIndex, DefIndex>, Vec<LocalDefId>) {
+    let defs = tcx.untracked().definitions.read();
+
+    let mut to_remap = vec![];
+    let mut def_ids = vec![];
+
+    let start = tcx.definitions().last_deterministic_index().as_usize() + 1;
+    for idx in start..defs.num_definitions() {
+        let def_id = LocalDefId { local_def_index: idx.into() };
+
+        to_remap.push((def_id, defs.def_path_hash(def_id).local_hash()));
+        def_ids.push(def_id);
+    }
+
+    to_remap.sort_by_key(|(_, hash)| *hash);
+
+    let mut remapping = FxHashMap::default();
+    for ((orig_id, _), remapped_id) in to_remap.into_iter().zip(def_ids) {
+        remapping.insert(orig_id.local_def_index, remapped_id.local_def_index);
+    }
+
+    let mut sorted_def_ids = (0..defs.num_definitions())
+        .into_iter()
+        .map(|idx| LocalDefId { local_def_index: DefIndex::new(idx) })
+        .collect::<Vec<_>>();
+
+    sorted_def_ids.sort_by_key(|id| {
+        let def_index = id.local_def_index;
+        remapping.get(&def_index).copied().unwrap_or(def_index)
+    });
+
+    (remapping, sorted_def_ids)
+}
+
 fn with_encode_metadata_header(
     tcx: TyCtxt<'_>,
     path: &Path,
@@ -2707,6 +2783,8 @@ fn with_encode_metadata_header(
         is_proc_macro: tcx.crate_types().contains(&CrateType::ProcMacro),
         hygiene_ctxt: Default::default(),
         symbol_index_table: Default::default(),
+        def_indexes_remapping: Default::default(),
+        last_deterministic_index: tcx.definitions().last_deterministic_index().as_u32(),
     };
 
     // Encode the rustc version string in a predictable location.

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -378,7 +378,7 @@ pub(crate) struct CrateDep {
 
 #[derive(MetadataEncodable, LazyDecodable)]
 pub(crate) struct TraitImpls {
-    trait_id: (u32, DefIndex),
+    trait_id: (u32 /* crate */, u32 /* def index */),
     impls: LazyArray<(DefIndex, Option<SimplifiedType>)>,
 }
 

tests/run-make/parallel-reproducible-build/async-fns-issue-162202.rs

@@ -0,0 +1,3 @@
+async fn test1(_: &'_ u8) {}
+async fn test2<'s>(_: &'s u8) {}
+fn main() {}

tests/run-make/parallel-reproducible-build/rmake.rs

@@ -14,6 +14,39 @@ fn main() {
     const TESTS: &[(&str, &[&str])] = &[
         ("static-muts-issue-140413", &["-Zthreads=50"]),
         ("derives-issue-129094", &["-Zthreads=16", "-Copt-level=3"]),
+        (
+            "async-fns-issue-162202",
+            &[
+                "-Zthreads=30",
+                "--crate-type=lib",
+                "--crate-name=test",
+                "-Ccodegen-units=1",
+                "-Zremap-cwd-prefix=reproducible_dir",
+                "--edition=2024",
+            ],
+        ),
+        (
+            "rpit-issue-162202",
+            &[
+                "-Zthreads=30",
+                "--crate-type=lib",
+                "--crate-name=test",
+                "-Ccodegen-units=1",
+                "-Zremap-cwd-prefix=reproducible_dir",
+                "--edition=2024",
+            ],
+        ),
+        (
+            "static-muts-lib-issue-162203",
+            &[
+                "-Zthreads=30",
+                "--edition=2024",
+                "-Zremap-cwd-prefix=reproducible_dir",
+                "--cap-lints=warn",
+                "-Ccodegen-units=1",
+                "-Clink-dead-code=true",
+            ],
+        ),
     ];
 
     for (file, args) in TESTS {