1//! This module specifies the type based interner for constants.2//!3//! After a const evaluation has computed a value, before we destroy the const evaluator's session4//! memory, we need to extract all memory allocations to the global memory pool so they stay around.5//!6//! In principle, this is not very complicated: we recursively walk the final value, follow all the7//! pointers, and move all reachable allocations to the global `tcx` memory. The only complication8//! is picking the right mutability: the outermost allocation generally has a clear mutability, but9//! what about the other allocations it points to that have also been created with this value? We10//! don't want to do guesswork here. The rules are: `static`, `const`, and promoted can only create11//! immutable allocations that way. `static mut` can be initialized with expressions like `&mut 42`,12//! so all inner allocations are marked mutable. Some of them could potentially be made immutable,13//! but that would require relying on type information, and given how many ways Rust has to lie14//! about type information, we want to avoid doing that.1516use hir::def::DefKind;17use rustc_ast::Mutability;18use rustc_data_structures::fx::{FxHashSet, FxIndexMap};19use rustc_hir::definitions::{DefPathData, PerParentDisambiguatorState};20use rustc_hir::{self as hir};21use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;22use rustc_middle::mir::interpret::{23    AllocBytes, ConstAllocation, CtfeProvenance, InterpResult, Provenance,24};25use rustc_middle::query::TyCtxtAt;26use rustc_middle::span_bug;27use rustc_middle::ty::TyCtxt;28use rustc_middle::ty::layout::TyAndLayout;29use rustc_span::def_id::LocalDefId;30use tracing::{instrument, trace};3132use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, interp_ok};33use crate::const_eval::DummyMachine;34use crate::{const_eval, errors};3536pub trait CompileTimeMachine<'tcx> = Machine<37        'tcx,38        MemoryKind = const_eval::MemoryKind,39        Provenance = CtfeProvenance,40        ExtraFnVal = !,41        FrameExtra = (),42        AllocExtra = (),43        MemoryMap = FxIndexMap<AllocId, (MemoryKind<const_eval::MemoryKind>, Allocation)>,44    > + HasStaticRootDefId;4546pub trait HasStaticRootDefId {47    /// Returns the `DefId` of the static item that is currently being evaluated.48    /// Used for interning to be able to handle nested allocations.49    fn static_def_id(&self) -> Option<LocalDefId>;50}5152impl HasStaticRootDefId for const_eval::CompileTimeMachine<'_> {53    fn static_def_id(&self) -> Option<LocalDefId> {54        Some(self.static_root_ids?.1)55    }56}5758fn prepare_alloc<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>(59    tcx: TyCtxt<'tcx>,60    kind: MemoryKind<const_eval::MemoryKind>,61    alloc: &mut Allocation<Prov, Extra, Bytes>,62    mutability: Mutability,63) -> Result<(), InternError> {64    match kind {65        MemoryKind::Machine(const_eval::MemoryKind::Heap { was_made_global }) => {66            if !was_made_global {67                // Attempting to intern a `const_allocate`d pointer that was not made global via68                // `const_make_global`.69                tcx.dcx().delayed_bug("non-global heap allocation in const value");70                return Err(InternError::ConstAllocNotGlobal);71            }72        }73        MemoryKind::Stack | MemoryKind::CallerLocation => {}74    }7576    if !alloc.provenance_merge_bytes(&tcx) {77        // Per-byte provenance is not supported by backends, so we cannot accept it here.78        tcx.dcx().delayed_bug("partial pointer in const value");79        return Err(InternError::PartialPointer);80    }8182    // Set allocation mutability as appropriate. This is used by LLVM to put things into83    // read-only memory, and also by Miri when evaluating other globals that84    // access this one.85    match mutability {86        Mutability::Not => {87            alloc.mutability = Mutability::Not;88        }89        Mutability::Mut => {90            // This must be already mutable, we won't "un-freeze" allocations ever.91            assert_eq!(alloc.mutability, Mutability::Mut);92        }93    }94    Ok(())95}9697/// Intern an allocation. Returns `Err` if the allocation does not exist in the local memory.98///99/// `mutability` can be used to force immutable interning: if it is `Mutability::Not`, the100/// allocation is interned immutably; if it is `Mutability::Mut`, then the allocation *must be*101/// already mutable (as a sanity check).102///103/// Returns an iterator over all relocations referred to by this allocation.104fn intern_shallow<'tcx, M: CompileTimeMachine<'tcx>>(105    ecx: &mut InterpCx<'tcx, M>,106    alloc_id: AllocId,107    mutability: Mutability,108    disambiguator: Option<&mut PerParentDisambiguatorState>,109) -> Result<impl Iterator<Item = CtfeProvenance> + 'tcx, InternError> {110    trace!("intern_shallow {:?}", alloc_id);111    // remove allocation112    // FIXME(#120456) - is `swap_remove` correct?113    let Some((kind, mut alloc)) = ecx.memory.alloc_map.swap_remove(&alloc_id) else {114        return Err(InternError::DanglingPointer);115    };116117    if let Err(err) = prepare_alloc(*ecx.tcx, kind, &mut alloc, mutability) {118        // We want to error here, but we have to first put the119        // allocation back into the `alloc_map` to keep things in a consistent state.120        ecx.memory.alloc_map.insert(alloc_id, (kind, alloc));121        return Err(err);122    }123124    // link the alloc id to the actual allocation125    let alloc = ecx.tcx.mk_const_alloc(alloc);126    if let Some(static_id) = ecx.machine.static_def_id() {127        intern_as_new_static(128            ecx.tcx,129            static_id,130            alloc_id,131            alloc,132            disambiguator.expect("disambiguator needed"),133        );134    } else {135        ecx.tcx.set_alloc_id_memory(alloc_id, alloc);136    }137    Ok(alloc.inner().provenance().ptrs().iter().map(|&(_, prov)| prov))138}139140/// Creates a new `DefId` and feeds all the right queries to make this `DefId`141/// appear as if it were a user-written `static` (though it has no HIR).142fn intern_as_new_static<'tcx>(143    tcx: TyCtxtAt<'tcx>,144    static_id: LocalDefId,145    alloc_id: AllocId,146    alloc: ConstAllocation<'tcx>,147    disambiguator: &mut PerParentDisambiguatorState,148) {149    // `intern_const_alloc_recursive` is called once per static and it contains the `PerParentDisambiguatorState`.150    //  The `<static_id>::{{nested}}` path is thus unique to `intern_const_alloc_recursive` and the151    // `PerParentDisambiguatorState` ensures the generated path is unique for this call as we generate152    // `<static_id>::{{nested#n}}` where `n` is the `n`th `intern_as_new_static` call.153    let feed = tcx.create_def(154        static_id,155        None,156        DefKind::Static { safety: hir::Safety::Safe, mutability: alloc.0.mutability, nested: true },157        Some(DefPathData::NestedStatic),158        disambiguator,159    );160    tcx.set_nested_alloc_id_static(alloc_id, feed.def_id());161162    if tcx.is_thread_local_static(static_id.into()) {163        tcx.dcx().emit_err(errors::NestedStaticInThreadLocal { span: tcx.def_span(static_id) });164    }165166    // These do not inherit the codegen attrs of the parent static allocation, since167    // it doesn't make sense for them to inherit their `#[no_mangle]` and `#[link_name = ..]`168    // and the like.169    feed.codegen_fn_attrs(CodegenFnAttrs::new());170171    feed.eval_static_initializer(Ok(alloc));172    feed.generics_of(tcx.generics_of(static_id).clone());173    feed.def_ident_span(tcx.def_ident_span(static_id));174    feed.explicit_predicates_of(tcx.explicit_predicates_of(static_id));175    feed.feed_hir();176}177178/// How a constant value should be interned.179#[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]180pub enum InternKind {181    /// The `mutability` of the static, ignoring the type which may have interior mutability.182    Static(hir::Mutability),183    /// A `const` item184    Constant,185    Promoted,186}187188#[derive(Debug)]189pub enum InternError {190    BadMutablePointer,191    DanglingPointer,192    ConstAllocNotGlobal,193    PartialPointer,194}195196/// Intern `ret` and everything it references.197///198/// This *cannot raise an interpreter error*. Doing so is left to validation, which199/// tracks where in the value we are and thus can show much better error messages.200///201/// For `InternKind::Static` the root allocation will not be interned, but must be handled by the caller.202#[instrument(level = "debug", skip(ecx))]203pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx>>(204    ecx: &mut InterpCx<'tcx, M>,205    intern_kind: InternKind,206    ret: &MPlaceTy<'tcx>,207) -> Result<(), InternError> {208    let mut disambiguator =209        ecx.machine.static_def_id().map(|id| PerParentDisambiguatorState::new(id));210    let mut disambiguator = disambiguator.as_mut();211212    // We are interning recursively, and for mutability we are distinguishing the "root" allocation213    // that we are starting in, and all other allocations that we are encountering recursively.214    let (base_mutability, inner_mutability, is_static) = match intern_kind {215        InternKind::Constant | InternKind::Promoted => {216            // Completely immutable. Interning anything mutably here can only lead to unsoundness,217            // since all consts are conceptually independent values but share the same underlying218            // memory.219            (Mutability::Not, Mutability::Not, false)220        }221        InternKind::Static(Mutability::Not) => {222            (223                // Outermost allocation is mutable if `!Freeze` i.e. contains interior mutable types.224                if ret.layout.ty.is_freeze(*ecx.tcx, ecx.typing_env) {225                    Mutability::Not226                } else {227                    Mutability::Mut228                },229                // Inner allocations are never mutable. They can only arise via the "tail230                // expression" / "outer scope" rule, and we treat them consistently with `const`.231                Mutability::Not,232                true,233            )234        }235        InternKind::Static(Mutability::Mut) => {236            // Just make everything mutable. We accept code like237            // `static mut X = &mut [42]`, so even inner allocations need to be mutable.238            (Mutability::Mut, Mutability::Mut, true)239        }240    };241242    // Intern the base allocation, and initialize todo list for recursive interning.243    let base_alloc_id = ret.ptr().provenance.unwrap().alloc_id();244    trace!(?base_alloc_id, ?base_mutability);245    // First we intern the base allocation, as it requires a different mutability.246    // This gives us the initial set of nested allocations, which will then all be processed247    // recursively in the loop below.248    let mut todo: Vec<_> = if is_static {249        assert!(disambiguator.is_some());250        // Do not steal the root allocation, we need it later to create the return value of `eval_static_initializer`.251        // But still change its mutability to match the requested one.252        let (kind, alloc) = ecx.memory.alloc_map.get_mut(&base_alloc_id).unwrap();253        prepare_alloc(*ecx.tcx, *kind, alloc, base_mutability)?;254        alloc.provenance().ptrs().iter().map(|&(_, prov)| prov).collect()255    } else {256        assert!(disambiguator.is_none());257        intern_shallow(ecx, base_alloc_id, base_mutability, None)?.collect()258    };259    // We need to distinguish "has just been interned" from "was already in `tcx`",260    // so we track this in a separate set.261    let mut just_interned: FxHashSet<_> = std::iter::once(base_alloc_id).collect();262    // Whether we encountered a bad mutable pointer.263    // We want to first report "dangling" and then "mutable", so we need to delay reporting these264    // errors.265    let mut found_bad_mutable_ptr = false;266267    // Keep interning as long as there are things to intern.268    // We show errors if there are dangling pointers, or mutable pointers in immutable contexts269    // (i.e., everything except for `static mut`). We only return these errors as a `Result`270    // so that the caller can run validation, and subsequently only report interning errors271    // if validation fails. Validation has the better error messages so we prefer those, but272    // interning has better coverage since it "sees" *all* pointers, including raw pointers and273    // references stored in unions.274    while let Some(prov) = todo.pop() {275        trace!(?prov);276        let alloc_id = prov.alloc_id();277278        if base_alloc_id == alloc_id && is_static {279            // This is a pointer to the static itself. It's ok for a static to refer to itself,280            // even mutably. Whether that mutable pointer is legal at all is checked in validation.281            // See tests/ui/statics/recursive_interior_mut.rs for how such a situation can occur.282            // We also already collected all the nested allocations, so there's no need to do that again.283            continue;284        }285286        // Ensure that this is derived from a shared reference. Crucially, we check this *before*287        // checking whether the `alloc_id` has already been interned. The point of this check is to288        // ensure that when there are multiple pointers to the same allocation, they are *all*289        // derived from a shared reference. Therefore it would be bad if we only checked the first290        // pointer to any given allocation.291        // (It is likely not possible to actually have multiple pointers to the same allocation,292        // so alternatively we could also check that and ICE if there are multiple such pointers.)293        // See <https://github.com/rust-lang/rust/pull/128543> for why we are checking for "shared294        // reference" and not "immutable", i.e., for why we are allowing interior-mutable shared295        // references: they can actually be created in safe code while pointing to apparently296        // "immutable" values, via promotion or tail expression lifetime extension of297        // `&None::<Cell<T>>`.298        // We also exclude promoteds from this as `&mut []` can be promoted, which is a mutable299        // reference pointing to an immutable (zero-sized) allocation. We rely on the promotion300        // analysis not screwing up to ensure that it is sound to intern promoteds as immutable.301        if intern_kind != InternKind::Promoted302            && inner_mutability == Mutability::Not303            && !prov.shared_ref()304        {305            let is_already_global = ecx.tcx.try_get_global_alloc(alloc_id).is_some();306            if is_already_global && !just_interned.contains(&alloc_id) {307                // This is a pointer to some memory from another constant. We encounter mutable308                // pointers to such memory since we do not always track immutability through309                // these "global" pointers. Allowing them is harmless; the point of these checks310                // during interning is to justify why we intern the *new* allocations immutably,311                // so we can completely ignore existing allocations.312                // We can also skip the rest of this loop iteration, since after all it is already313                // interned.314                continue;315            }316            // If this is a dangling pointer, that's actually fine -- the problematic case is317            // when there is memory there that someone might expect to be mutable, but we make it immutable.318            let dangling = !is_already_global && !ecx.memory.alloc_map.contains_key(&alloc_id);319            if !dangling {320                found_bad_mutable_ptr = true;321            }322        }323        if ecx.tcx.try_get_global_alloc(alloc_id).is_some() {324            // Already interned.325            debug_assert!(!ecx.memory.alloc_map.contains_key(&alloc_id));326            continue;327        }328        // We always intern with `inner_mutability`, and furthermore we ensured above that if329        // that is "immutable", then there are *no* mutable pointers anywhere in the newly330        // interned memory -- justifying that we can indeed intern immutably. However this also331        // means we can *not* easily intern immutably here if `prov.immutable()` is true and332        // `inner_mutability` is `Mut`: there might be other pointers to that allocation, and333        // we'd have to somehow check that they are *all* immutable before deciding that this334        // allocation can be made immutable. In the future we could consider analyzing all335        // pointers before deciding which allocations can be made immutable; but for now we are336        // okay with losing some potential for immutability here. This can anyway only affect337        // `static mut`.338        just_interned.insert(alloc_id);339        let next = intern_shallow(ecx, alloc_id, inner_mutability, disambiguator.as_deref_mut())?;340        todo.extend(next);341    }342    if found_bad_mutable_ptr {343        // We found a mutable pointer inside a const where inner allocations should be immutable,344        // and there was no other error. This should usually never happen! However, this can happen345        // in unleash-miri mode, so report it as a normal error then.346        if ecx.tcx.sess.opts.unstable_opts.unleash_the_miri_inside_of_you {347            return Err(InternError::BadMutablePointer);348        } else {349            span_bug!(350                ecx.tcx.span,351                "the static const safety checks accepted a mutable pointer they should not have accepted"352            );353        }354    }355    Ok(())356}357358/// Intern `ret`. This function assumes that `ret` references no other allocation.359#[instrument(level = "debug", skip(ecx))]360pub fn intern_const_alloc_for_constprop<'tcx, M: CompileTimeMachine<'tcx>>(361    ecx: &mut InterpCx<'tcx, M>,362    alloc_id: AllocId,363) -> InterpResult<'tcx, ()> {364    if ecx.tcx.try_get_global_alloc(alloc_id).is_some() {365        // The constant is already in global memory. Do nothing.366        return interp_ok(());367    }368    // Move allocation to `tcx`.369    if let Some(_) = intern_shallow(ecx, alloc_id, Mutability::Not, None).unwrap().next() {370        // We are not doing recursive interning, so we don't currently support provenance.371        // (If this assertion ever triggers, we should just implement a372        // proper recursive interning loop -- or just call `intern_const_alloc_recursive`.373        panic!("`intern_const_alloc_for_constprop` called on allocation with nested provenance")374    }375    interp_ok(())376}377378impl<'tcx> InterpCx<'tcx, DummyMachine> {379    /// A helper function that allocates memory for the layout given and gives you access to mutate380    /// it. Once your own mutation code is done, the backing `Allocation` is removed from the381    /// current `Memory` and interned as read-only into the global memory.382    pub fn intern_with_temp_alloc(383        &mut self,384        layout: TyAndLayout<'tcx>,385        f: impl FnOnce(386            &mut InterpCx<'tcx, DummyMachine>,387            &PlaceTy<'tcx, CtfeProvenance>,388        ) -> InterpResult<'tcx, ()>,389    ) -> InterpResult<'tcx, AllocId> {390        // `allocate` picks a fresh AllocId that we will associate with its data below.391        let dest = self.allocate(layout, MemoryKind::Stack)?;392        f(self, &dest.clone().into())?;393        let alloc_id = dest.ptr().provenance.unwrap().alloc_id(); // this was just allocated, it must have provenance394        for prov in intern_shallow(self, alloc_id, Mutability::Not, None).unwrap() {395            // We are not doing recursive interning, so we don't currently support provenance.396            // (If this assertion ever triggers, we should just implement a397            // proper recursive interning loop -- or just call `intern_const_alloc_recursive`.398            if self.tcx.try_get_global_alloc(prov.alloc_id()).is_none() {399                panic!("`intern_with_temp_alloc` with nested allocations");400            }401        }402        interp_ok(alloc_id)403    }404}