1use rustc_data_structures::fx::FxIndexMap;2use rustc_index::bit_set::{DenseBitSet, MixedBitSet};3use rustc_middle::mir::{self, BasicBlock, Body, CallReturnPlaces, Location, Place};4use rustc_middle::ty::{RegionVid, TyCtxt};5use rustc_mir_dataflow::fmt::DebugWithContext;6use rustc_mir_dataflow::impls::{7    EverInitializedPlaces, EverInitializedPlacesDomain, MaybeUninitializedPlaces,8    MaybeUninitializedPlacesDomain,9};10use rustc_mir_dataflow::{Analysis, GenKill, JoinSemiLattice};11use tracing::debug;1213use crate::{BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext, places_conflict};1415// This analysis is different to most others. Its results aren't computed with16// `iterate_to_fixpoint`, but are instead composed from the results of three sub-analyses that are17// computed individually with `iterate_to_fixpoint`. Because it's faster that way than having a18// single analysis where the domain has three components.19pub(crate) struct Borrowck<'a, 'tcx> {20    pub(crate) borrows: Borrows<'a, 'tcx>,21    pub(crate) uninits: MaybeUninitializedPlaces<'a, 'tcx>,22    pub(crate) ever_inits: EverInitializedPlaces<'a, 'tcx>,23}2425impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> {26    type Domain = BorrowckDomain;2728    const NAME: &'static str = "borrowck";2930    fn bottom_value(&self, body: &mir::Body<'tcx>) -> Self::Domain {31        BorrowckDomain {32            borrows: self.borrows.bottom_value(body),33            uninits: self.uninits.bottom_value(body),34            ever_inits: self.ever_inits.bottom_value(body),35        }36    }3738    fn initialize_start_block(&self, _body: &mir::Body<'tcx>, _state: &mut Self::Domain) {39        // This is only reachable from `iterate_to_fixpoint`, which this analysis doesn't use.40        unreachable!();41    }4243    fn apply_early_statement_effect(44        &self,45        state: &mut Self::Domain,46        stmt: &mir::Statement<'tcx>,47        loc: Location,48    ) {49        self.borrows.apply_early_statement_effect(&mut state.borrows, stmt, loc);50        self.uninits.apply_early_statement_effect(&mut state.uninits, stmt, loc);51        self.ever_inits.apply_early_statement_effect(&mut state.ever_inits, stmt, loc);52    }5354    fn apply_primary_statement_effect(55        &self,56        state: &mut Self::Domain,57        stmt: &mir::Statement<'tcx>,58        loc: Location,59    ) {60        self.borrows.apply_primary_statement_effect(&mut state.borrows, stmt, loc);61        self.uninits.apply_primary_statement_effect(&mut state.uninits, stmt, loc);62        self.ever_inits.apply_primary_statement_effect(&mut state.ever_inits, stmt, loc);63    }6465    fn apply_early_terminator_effect(66        &self,67        state: &mut Self::Domain,68        term: &mir::Terminator<'tcx>,69        loc: Location,70    ) {71        self.borrows.apply_early_terminator_effect(&mut state.borrows, term, loc);72        self.uninits.apply_early_terminator_effect(&mut state.uninits, term, loc);73        self.ever_inits.apply_early_terminator_effect(&mut state.ever_inits, term, loc);74    }7576    fn apply_primary_terminator_effect(77        &self,78        state: &mut Self::Domain,79        term: &mir::Terminator<'tcx>,80        loc: Location,81    ) {82        self.borrows.apply_primary_terminator_effect(&mut state.borrows, term, loc);83        self.uninits.apply_primary_terminator_effect(&mut state.uninits, term, loc);84        self.ever_inits.apply_primary_terminator_effect(&mut state.ever_inits, term, loc);85    }8687    fn apply_call_return_effect(88        &self,89        _state: &mut Self::Domain,90        _block: BasicBlock,91        _return_places: CallReturnPlaces<'_, 'tcx>,92    ) {93        // This is only reachable from `iterate_to_fixpoint`, which this analysis doesn't use.94        unreachable!();95    }96}9798impl JoinSemiLattice for BorrowckDomain {99    fn join(&mut self, _other: &Self) -> bool {100        // This is only reachable from `iterate_to_fixpoint`, which this analysis doesn't use.101        unreachable!();102    }103}104105/// The transient state of the dataflow analyses used by the borrow checker.106#[derive(Clone, Debug, PartialEq, Eq)]107pub(crate) struct BorrowckDomain {108    pub(crate) borrows: BorrowsDomain,109    pub(crate) uninits: MaybeUninitializedPlacesDomain,110    pub(crate) ever_inits: EverInitializedPlacesDomain,111}112113rustc_index::newtype_index! {114    #[orderable]115    #[debug_format = "bw{}"]116    pub struct BorrowIndex {}117}118119/// `Borrows` stores the data used in the analyses that track the flow120/// of borrows.121///122/// It uniquely identifies every borrow (`Rvalue::Ref`) by a123/// `BorrowIndex`, and maps each such index to a `BorrowData`124/// describing the borrow. These indexes are used for representing the125/// borrows in compact bitvectors.126pub struct Borrows<'a, 'tcx> {127    tcx: TyCtxt<'tcx>,128    body: &'a Body<'tcx>,129    borrow_set: &'a BorrowSet<'tcx>,130    borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,131}132133struct OutOfScopePrecomputer<'a, 'tcx> {134    visited: DenseBitSet<mir::BasicBlock>,135    visit_stack: Vec<mir::BasicBlock>,136    body: &'a Body<'tcx>,137    regioncx: &'a RegionInferenceContext<'tcx>,138    borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,139}140141impl<'tcx> OutOfScopePrecomputer<'_, 'tcx> {142    fn compute(143        body: &Body<'tcx>,144        regioncx: &RegionInferenceContext<'tcx>,145        borrow_set: &BorrowSet<'tcx>,146    ) -> FxIndexMap<Location, Vec<BorrowIndex>> {147        let mut prec = OutOfScopePrecomputer {148            visited: DenseBitSet::new_empty(body.basic_blocks.len()),149            visit_stack: vec![],150            body,151            regioncx,152            borrows_out_of_scope_at_location: FxIndexMap::default(),153        };154        for (borrow_index, borrow_data) in borrow_set.iter_enumerated() {155            let borrow_region = borrow_data.region;156            let location = borrow_data.reserve_location;157            prec.precompute_borrows_out_of_scope(borrow_index, borrow_region, location);158        }159160        prec.borrows_out_of_scope_at_location161    }162163    fn precompute_borrows_out_of_scope(164        &mut self,165        borrow_index: BorrowIndex,166        borrow_region: RegionVid,167        first_location: Location,168    ) {169        let first_block = first_location.block;170        let first_bb_data = &self.body.basic_blocks[first_block];171172        // This is the first block, we only want to visit it from the creation of the borrow at173        // `first_location`.174        let first_lo = first_location.statement_index;175        let first_hi = first_bb_data.statements.len();176177        if let Some(kill_stmt) = self.regioncx.first_non_contained_inclusive(178            borrow_region,179            first_block,180            first_lo,181            first_hi,182        ) {183            let kill_location = Location { block: first_block, statement_index: kill_stmt };184            // If region does not contain a point at the location, then add to list and skip185            // successor locations.186            debug!("borrow {:?} gets killed at {:?}", borrow_index, kill_location);187            self.borrows_out_of_scope_at_location188                .entry(kill_location)189                .or_default()190                .push(borrow_index);191192            // The borrow is already dead, there is no need to visit other blocks.193            return;194        }195196        // The borrow is not dead. Add successor BBs to the work list, if necessary.197        for succ_bb in first_bb_data.terminator().successors() {198            if self.visited.insert(succ_bb) {199                self.visit_stack.push(succ_bb);200            }201        }202203        // We may end up visiting `first_block` again. This is not an issue: we know at this point204        // that it does not kill the borrow in the `first_lo..=first_hi` range, so checking the205        // `0..first_lo` range and the `0..first_hi` range give the same result.206        while let Some(block) = self.visit_stack.pop() {207            let bb_data = &self.body[block];208            let num_stmts = bb_data.statements.len();209            if let Some(kill_stmt) =210                self.regioncx.first_non_contained_inclusive(borrow_region, block, 0, num_stmts)211            {212                let kill_location = Location { block, statement_index: kill_stmt };213                // If region does not contain a point at the location, then add to list and skip214                // successor locations.215                debug!("borrow {:?} gets killed at {:?}", borrow_index, kill_location);216                self.borrows_out_of_scope_at_location217                    .entry(kill_location)218                    .or_default()219                    .push(borrow_index);220221                // We killed the borrow, so we do not visit this block's successors.222                continue;223            }224225            // Add successor BBs to the work list, if necessary.226            for succ_bb in bb_data.terminator().successors() {227                if self.visited.insert(succ_bb) {228                    self.visit_stack.push(succ_bb);229                }230            }231        }232233        self.visited.clear();234    }235}236237// This is `pub` because it's used by unstable external borrowck data users, see `consumers.rs`.238pub fn calculate_borrows_out_of_scope_at_location<'tcx>(239    body: &Body<'tcx>,240    regioncx: &RegionInferenceContext<'tcx>,241    borrow_set: &BorrowSet<'tcx>,242) -> FxIndexMap<Location, Vec<BorrowIndex>> {243    OutOfScopePrecomputer::compute(body, regioncx, borrow_set)244}245246struct PoloniusOutOfScopePrecomputer<'a, 'tcx> {247    visited: DenseBitSet<mir::BasicBlock>,248    visit_stack: Vec<mir::BasicBlock>,249    body: &'a Body<'tcx>,250    regioncx: &'a RegionInferenceContext<'tcx>,251252    loans_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,253}254255impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {256    fn compute(257        body: &Body<'tcx>,258        regioncx: &RegionInferenceContext<'tcx>,259        borrow_set: &BorrowSet<'tcx>,260    ) -> FxIndexMap<Location, Vec<BorrowIndex>> {261        // The in-tree polonius analysis computes loans going out of scope using the262        // set-of-loans model.263        let mut prec = PoloniusOutOfScopePrecomputer {264            visited: DenseBitSet::new_empty(body.basic_blocks.len()),265            visit_stack: vec![],266            body,267            regioncx,268            loans_out_of_scope_at_location: FxIndexMap::default(),269        };270        for (loan_idx, loan_data) in borrow_set.iter_enumerated() {271            let loan_issued_at = loan_data.reserve_location;272            prec.precompute_loans_out_of_scope(loan_idx, loan_issued_at);273        }274275        prec.loans_out_of_scope_at_location276    }277278    /// Loans are in scope while they are live: whether they are contained within any live region.279    /// In the location-insensitive analysis, a loan will be contained in a region if the issuing280    /// region can reach it in the subset graph. So this is a reachability problem.281    fn precompute_loans_out_of_scope(&mut self, loan_idx: BorrowIndex, loan_issued_at: Location) {282        let first_block = loan_issued_at.block;283        let first_bb_data = &self.body.basic_blocks[first_block];284285        // The first block we visit is the one where the loan is issued, starting from the statement286        // where the loan is issued: at `loan_issued_at`.287        let first_lo = loan_issued_at.statement_index;288        let first_hi = first_bb_data.statements.len();289290        if let Some(kill_location) =291            self.loan_kill_location(loan_idx, loan_issued_at, first_block, first_lo, first_hi)292        {293            debug!("loan {:?} gets killed at {:?}", loan_idx, kill_location);294            self.loans_out_of_scope_at_location.entry(kill_location).or_default().push(loan_idx);295296            // The loan dies within the first block, we're done and can early return.297            return;298        }299300        // The loan is not dead. Add successor BBs to the work list, if necessary.301        for succ_bb in first_bb_data.terminator().successors() {302            if self.visited.insert(succ_bb) {303                self.visit_stack.push(succ_bb);304            }305        }306307        // We may end up visiting `first_block` again. This is not an issue: we know at this point308        // that the loan is not killed in the `first_lo..=first_hi` range, so checking the309        // `0..first_lo` range and the `0..first_hi` range gives the same result.310        while let Some(block) = self.visit_stack.pop() {311            let bb_data = &self.body[block];312            let num_stmts = bb_data.statements.len();313            if let Some(kill_location) =314                self.loan_kill_location(loan_idx, loan_issued_at, block, 0, num_stmts)315            {316                debug!("loan {:?} gets killed at {:?}", loan_idx, kill_location);317                self.loans_out_of_scope_at_location318                    .entry(kill_location)319                    .or_default()320                    .push(loan_idx);321322                // The loan dies within this block, so we don't need to visit its successors.323                continue;324            }325326            // Add successor BBs to the work list, if necessary.327            for succ_bb in bb_data.terminator().successors() {328                if self.visited.insert(succ_bb) {329                    self.visit_stack.push(succ_bb);330                }331            }332        }333334        self.visited.clear();335        assert!(self.visit_stack.is_empty(), "visit stack should be empty");336    }337338    /// Returns the lowest statement in `start..=end`, where the loan goes out of scope, if any.339    /// This is the statement where the issuing region can't reach any of the regions that are live340    /// at this point.341    fn loan_kill_location(342        &self,343        loan_idx: BorrowIndex,344        loan_issued_at: Location,345        block: BasicBlock,346        start: usize,347        end: usize,348    ) -> Option<Location> {349        for statement_index in start..=end {350            let location = Location { block, statement_index };351352            // Check whether the issuing region can reach local regions that are live at this point:353            // - a loan is always live at its issuing location because it can reach the issuing354            // region, which is always live at this location.355            if location == loan_issued_at {356                continue;357            }358359            // - the loan goes out of scope at `location` if it's not contained within any regions360            // live at this point.361            //362            // FIXME: if the issuing region `i` can reach a live region `r` at point `p`, and `r` is363            // live at point `q`, then it's guaranteed that `i` would reach `r` at point `q`.364            // Reachability is location-insensitive, and we could take advantage of that, by jumping365            // to a further point than just the next statement: we can jump to the furthest point366            // within the block where `r` is live.367            if self.regioncx.is_loan_live_at(loan_idx, location) {368                continue;369            }370371            // No live region is reachable from the issuing region: the loan is killed at this372            // point.373            return Some(location);374        }375376        None377    }378}379380impl<'a, 'tcx> Borrows<'a, 'tcx> {381    pub fn new(382        tcx: TyCtxt<'tcx>,383        body: &'a Body<'tcx>,384        regioncx: &RegionInferenceContext<'tcx>,385        borrow_set: &'a BorrowSet<'tcx>,386    ) -> Self {387        let borrows_out_of_scope_at_location =388            if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {389                calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set)390            } else {391                PoloniusOutOfScopePrecomputer::compute(body, regioncx, borrow_set)392            };393        Borrows { tcx, body, borrow_set, borrows_out_of_scope_at_location }394    }395396    /// Add all borrows to the kill set, if those borrows are out of scope at `location`.397    /// That means they went out of a nonlexical scope398    fn kill_loans_out_of_scope_at_location(399        &self,400        state: &mut <Self as Analysis<'tcx>>::Domain,401        location: Location,402    ) {403        // NOTE: The state associated with a given `location`404        // reflects the dataflow on entry to the statement.405        // Iterate over each of the borrows that we've precomputed406        // to have went out of scope at this location and kill them.407        //408        // We are careful always to call this function *before* we409        // set up the gen-bits for the statement or410        // terminator. That way, if the effect of the statement or411        // terminator *does* introduce a new loan of the same412        // region, then setting that gen-bit will override any413        // potential kill introduced here.414        if let Some(indices) = self.borrows_out_of_scope_at_location.get(&location) {415            state.kill_all(indices.iter().copied());416        }417    }418419    /// Kill any borrows that conflict with `place`.420    fn kill_borrows_on_place(421        &self,422        state: &mut <Self as Analysis<'tcx>>::Domain,423        place: Place<'tcx>,424    ) {425        debug!("kill_borrows_on_place: place={:?}", place);426427        let other_borrows_of_local = self428            .borrow_set429            .borrows_on_local(place.local)430            .map(|bs| bs.iter().copied())431            .into_flat_iter();432433        // If the borrowed place is a local with no projections, all other borrows of this434        // local must conflict. This is purely an optimization so we don't have to call435        // `places_conflict` for every borrow.436        if place.projection.is_empty() {437            if !self.body.local_decls[place.local].is_ref_to_static() {438                state.kill_all(other_borrows_of_local);439            }440            return;441        }442443        // By passing `PlaceConflictBias::NoOverlap`, we conservatively assume that any given444        // pair of array indices are not equal, so that when `places_conflict` returns true, we445        // will be assured that two places being compared definitely denotes the same sets of446        // locations.447        let definitely_conflicting_borrows = other_borrows_of_local.filter(|&i| {448            places_conflict(449                self.tcx,450                self.body,451                self.borrow_set[i].borrowed_place,452                place,453                PlaceConflictBias::NoOverlap,454            )455        });456457        state.kill_all(definitely_conflicting_borrows);458    }459}460461type BorrowsDomain = MixedBitSet<BorrowIndex>;462463/// Forward dataflow computation of the set of borrows that are in scope at a particular location.464/// - we gen the introduced loans465/// - we kill loans on locals going out of (regular) scope466/// - we kill the loans going out of their region's NLL scope: in NLL terms, the frontier where a467///   region stops containing the CFG points reachable from the issuing location.468/// - we also kill loans of conflicting places when overwriting a shared path: e.g. borrows of469///   `a.b.c` when `a` is overwritten.470impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> {471    type Domain = BorrowsDomain;472473    const NAME: &'static str = "borrows";474475    fn bottom_value(&self, _: &mir::Body<'tcx>) -> Self::Domain {476        // bottom = nothing is reserved or activated yet;477        MixedBitSet::new_empty(self.borrow_set.len())478    }479480    fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut Self::Domain) {481        // no borrows of code region_scopes have been taken prior to482        // function execution, so this method has no effect.483    }484485    fn apply_early_statement_effect(486        &self,487        state: &mut Self::Domain,488        _statement: &mir::Statement<'tcx>,489        location: Location,490    ) {491        self.kill_loans_out_of_scope_at_location(state, location);492    }493494    fn apply_primary_statement_effect(495        &self,496        state: &mut Self::Domain,497        stmt: &mir::Statement<'tcx>,498        location: Location,499    ) {500        match &stmt.kind {501            mir::StatementKind::Assign((lhs, rhs)) => {502                if let mir::Rvalue::Ref(_, _, place) | mir::Rvalue::Reborrow(_, _, place) = rhs {503                    if place.ignore_borrow(504                        self.tcx,505                        self.body,506                        &self.borrow_set.locals_state_at_exit(),507                    ) {508                        return;509                    }510                    let idxs =511                        self.borrow_set.borrows_at_location(&location).unwrap_or_else(|| {512                            panic!("could not find BorrowIndex for location {location:?}");513                        });514515                    for index in idxs {516                        state.gen_(*index);517                    }518                }519520                // Make sure there are no remaining borrows for variables521                // that are assigned over.522                self.kill_borrows_on_place(state, *lhs);523            }524525            mir::StatementKind::StorageDead(local) => {526                // Make sure there are no remaining borrows for locals that527                // are gone out of scope.528                self.kill_borrows_on_place(state, Place::from(*local));529            }530531            mir::StatementKind::FakeRead(..)532            | mir::StatementKind::SetDiscriminant { .. }533            | mir::StatementKind::StorageLive(..)534            | mir::StatementKind::PlaceMention(..)535            | mir::StatementKind::AscribeUserType(..)536            | mir::StatementKind::Coverage(..)537            | mir::StatementKind::Intrinsic(..)538            | mir::StatementKind::ConstEvalCounter539            | mir::StatementKind::BackwardIncompatibleDropHint { .. }540            | mir::StatementKind::Nop => {}541        }542    }543544    fn apply_early_terminator_effect(545        &self,546        state: &mut Self::Domain,547        _terminator: &mir::Terminator<'tcx>,548        location: Location,549    ) {550        self.kill_loans_out_of_scope_at_location(state, location);551    }552553    fn apply_primary_terminator_effect(554        &self,555        state: &mut Self::Domain,556        terminator: &mir::Terminator<'tcx>,557        _location: Location,558    ) {559        if let mir::TerminatorKind::InlineAsm { operands, .. } = &terminator.kind {560            for op in operands {561                if let mir::InlineAsmOperand::Out { place: Some(place), .. }562                | mir::InlineAsmOperand::InOut { out_place: Some(place), .. } = *op563                {564                    self.kill_borrows_on_place(state, place);565                }566            }567        }568    }569}570571impl<C> DebugWithContext<C> for BorrowIndex {}