1use std::fmt;23use rustc_data_structures::fx::FxIndexMap;4use rustc_index::bit_set::{DenseBitSet, MixedBitSet};5use rustc_middle::mir::{6    self, BasicBlock, Body, CallReturnPlaces, Location, Place, TerminatorEdges,7};8use rustc_middle::ty::{RegionVid, TyCtxt};9use rustc_mir_dataflow::fmt::DebugWithContext;10use rustc_mir_dataflow::impls::{11    EverInitializedPlaces, EverInitializedPlacesDomain, MaybeUninitializedPlaces,12    MaybeUninitializedPlacesDomain,13};14use rustc_mir_dataflow::{Analysis, GenKill, JoinSemiLattice};15use tracing::debug;1617use crate::{BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext, places_conflict};1819// This analysis is different to most others. Its results aren't computed with20// `iterate_to_fixpoint`, but are instead composed from the results of three sub-analyses that are21// computed individually with `iterate_to_fixpoint`.22pub(crate) struct Borrowck<'a, 'tcx> {23    pub(crate) borrows: Borrows<'a, 'tcx>,24    pub(crate) uninits: MaybeUninitializedPlaces<'a, 'tcx>,25    pub(crate) ever_inits: EverInitializedPlaces<'a, 'tcx>,26}2728impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> {29    type Domain = BorrowckDomain;3031    const NAME: &'static str = "borrowck";3233    fn bottom_value(&self, body: &mir::Body<'tcx>) -> Self::Domain {34        BorrowckDomain {35            borrows: self.borrows.bottom_value(body),36            uninits: self.uninits.bottom_value(body),37            ever_inits: self.ever_inits.bottom_value(body),38        }39    }4041    fn initialize_start_block(&self, _body: &mir::Body<'tcx>, _state: &mut Self::Domain) {42        // This is only reachable from `iterate_to_fixpoint`, which this analysis doesn't use.43        unreachable!();44    }4546    fn apply_early_statement_effect(47        &self,48        state: &mut Self::Domain,49        stmt: &mir::Statement<'tcx>,50        loc: Location,51    ) {52        self.borrows.apply_early_statement_effect(&mut state.borrows, stmt, loc);53        self.uninits.apply_early_statement_effect(&mut state.uninits, stmt, loc);54        self.ever_inits.apply_early_statement_effect(&mut state.ever_inits, stmt, loc);55    }5657    fn apply_primary_statement_effect(58        &self,59        state: &mut Self::Domain,60        stmt: &mir::Statement<'tcx>,61        loc: Location,62    ) {63        self.borrows.apply_primary_statement_effect(&mut state.borrows, stmt, loc);64        self.uninits.apply_primary_statement_effect(&mut state.uninits, stmt, loc);65        self.ever_inits.apply_primary_statement_effect(&mut state.ever_inits, stmt, loc);66    }6768    fn apply_early_terminator_effect(69        &self,70        state: &mut Self::Domain,71        term: &mir::Terminator<'tcx>,72        loc: Location,73    ) {74        self.borrows.apply_early_terminator_effect(&mut state.borrows, term, loc);75        self.uninits.apply_early_terminator_effect(&mut state.uninits, term, loc);76        self.ever_inits.apply_early_terminator_effect(&mut state.ever_inits, term, loc);77    }7879    fn apply_primary_terminator_effect<'mir>(80        &self,81        state: &mut Self::Domain,82        term: &'mir mir::Terminator<'tcx>,83        loc: Location,84    ) -> TerminatorEdges<'mir, 'tcx> {85        self.borrows.apply_primary_terminator_effect(&mut state.borrows, term, loc);86        self.uninits.apply_primary_terminator_effect(&mut state.uninits, term, loc);87        self.ever_inits.apply_primary_terminator_effect(&mut state.ever_inits, term, loc);8889        // This return value doesn't matter. It's only used by `iterate_to_fixpoint`, which this90        // analysis doesn't use.91        TerminatorEdges::None92    }9394    fn apply_call_return_effect(95        &self,96        _state: &mut Self::Domain,97        _block: BasicBlock,98        _return_places: CallReturnPlaces<'_, 'tcx>,99    ) {100        // This is only reachable from `iterate_to_fixpoint`, which this analysis doesn't use.101        unreachable!();102    }103}104105impl JoinSemiLattice for BorrowckDomain {106    fn join(&mut self, _other: &Self) -> bool {107        // This is only reachable from `iterate_to_fixpoint`, which this analysis doesn't use.108        unreachable!();109    }110}111112impl<'tcx, C> DebugWithContext<C> for BorrowckDomain113where114    C: rustc_mir_dataflow::move_paths::HasMoveData<'tcx>,115{116    fn fmt_with(&self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result {117        f.write_str("borrows: ")?;118        self.borrows.fmt_with(ctxt, f)?;119        f.write_str(" uninits: ")?;120        self.uninits.fmt_with(ctxt, f)?;121        f.write_str(" ever_inits: ")?;122        self.ever_inits.fmt_with(ctxt, f)?;123        Ok(())124    }125126    fn fmt_diff_with(&self, old: &Self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result {127        if self == old {128            return Ok(());129        }130131        if self.borrows != old.borrows {132            f.write_str("borrows: ")?;133            self.borrows.fmt_diff_with(&old.borrows, ctxt, f)?;134            f.write_str("\n")?;135        }136137        if self.uninits != old.uninits {138            f.write_str("uninits: ")?;139            self.uninits.fmt_diff_with(&old.uninits, ctxt, f)?;140            f.write_str("\n")?;141        }142143        if self.ever_inits != old.ever_inits {144            f.write_str("ever_inits: ")?;145            self.ever_inits.fmt_diff_with(&old.ever_inits, ctxt, f)?;146            f.write_str("\n")?;147        }148149        Ok(())150    }151}152153/// The transient state of the dataflow analyses used by the borrow checker.154#[derive(Clone, Debug, PartialEq, Eq)]155pub(crate) struct BorrowckDomain {156    pub(crate) borrows: BorrowsDomain,157    pub(crate) uninits: MaybeUninitializedPlacesDomain,158    pub(crate) ever_inits: EverInitializedPlacesDomain,159}160161rustc_index::newtype_index! {162    #[orderable]163    #[debug_format = "bw{}"]164    pub struct BorrowIndex {}165}166167/// `Borrows` stores the data used in the analyses that track the flow168/// of borrows.169///170/// It uniquely identifies every borrow (`Rvalue::Ref`) by a171/// `BorrowIndex`, and maps each such index to a `BorrowData`172/// describing the borrow. These indexes are used for representing the173/// borrows in compact bitvectors.174pub struct Borrows<'a, 'tcx> {175    tcx: TyCtxt<'tcx>,176    body: &'a Body<'tcx>,177    borrow_set: &'a BorrowSet<'tcx>,178    borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,179}180181struct OutOfScopePrecomputer<'a, 'tcx> {182    visited: DenseBitSet<mir::BasicBlock>,183    visit_stack: Vec<mir::BasicBlock>,184    body: &'a Body<'tcx>,185    regioncx: &'a RegionInferenceContext<'tcx>,186    borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,187}188189impl<'tcx> OutOfScopePrecomputer<'_, 'tcx> {190    fn compute(191        body: &Body<'tcx>,192        regioncx: &RegionInferenceContext<'tcx>,193        borrow_set: &BorrowSet<'tcx>,194    ) -> FxIndexMap<Location, Vec<BorrowIndex>> {195        let mut prec = OutOfScopePrecomputer {196            visited: DenseBitSet::new_empty(body.basic_blocks.len()),197            visit_stack: vec![],198            body,199            regioncx,200            borrows_out_of_scope_at_location: FxIndexMap::default(),201        };202        for (borrow_index, borrow_data) in borrow_set.iter_enumerated() {203            let borrow_region = borrow_data.region;204            let location = borrow_data.reserve_location;205            prec.precompute_borrows_out_of_scope(borrow_index, borrow_region, location);206        }207208        prec.borrows_out_of_scope_at_location209    }210211    fn precompute_borrows_out_of_scope(212        &mut self,213        borrow_index: BorrowIndex,214        borrow_region: RegionVid,215        first_location: Location,216    ) {217        let first_block = first_location.block;218        let first_bb_data = &self.body.basic_blocks[first_block];219220        // This is the first block, we only want to visit it from the creation of the borrow at221        // `first_location`.222        let first_lo = first_location.statement_index;223        let first_hi = first_bb_data.statements.len();224225        if let Some(kill_stmt) = self.regioncx.first_non_contained_inclusive(226            borrow_region,227            first_block,228            first_lo,229            first_hi,230        ) {231            let kill_location = Location { block: first_block, statement_index: kill_stmt };232            // If region does not contain a point at the location, then add to list and skip233            // successor locations.234            debug!("borrow {:?} gets killed at {:?}", borrow_index, kill_location);235            self.borrows_out_of_scope_at_location236                .entry(kill_location)237                .or_default()238                .push(borrow_index);239240            // The borrow is already dead, there is no need to visit other blocks.241            return;242        }243244        // The borrow is not dead. Add successor BBs to the work list, if necessary.245        for succ_bb in first_bb_data.terminator().successors() {246            if self.visited.insert(succ_bb) {247                self.visit_stack.push(succ_bb);248            }249        }250251        // We may end up visiting `first_block` again. This is not an issue: we know at this point252        // that it does not kill the borrow in the `first_lo..=first_hi` range, so checking the253        // `0..first_lo` range and the `0..first_hi` range give the same result.254        while let Some(block) = self.visit_stack.pop() {255            let bb_data = &self.body[block];256            let num_stmts = bb_data.statements.len();257            if let Some(kill_stmt) =258                self.regioncx.first_non_contained_inclusive(borrow_region, block, 0, num_stmts)259            {260                let kill_location = Location { block, statement_index: kill_stmt };261                // If region does not contain a point at the location, then add to list and skip262                // successor locations.263                debug!("borrow {:?} gets killed at {:?}", borrow_index, kill_location);264                self.borrows_out_of_scope_at_location265                    .entry(kill_location)266                    .or_default()267                    .push(borrow_index);268269                // We killed the borrow, so we do not visit this block's successors.270                continue;271            }272273            // Add successor BBs to the work list, if necessary.274            for succ_bb in bb_data.terminator().successors() {275                if self.visited.insert(succ_bb) {276                    self.visit_stack.push(succ_bb);277                }278            }279        }280281        self.visited.clear();282    }283}284285// This is `pub` because it's used by unstable external borrowck data users, see `consumers.rs`.286pub fn calculate_borrows_out_of_scope_at_location<'tcx>(287    body: &Body<'tcx>,288    regioncx: &RegionInferenceContext<'tcx>,289    borrow_set: &BorrowSet<'tcx>,290) -> FxIndexMap<Location, Vec<BorrowIndex>> {291    OutOfScopePrecomputer::compute(body, regioncx, borrow_set)292}293294struct PoloniusOutOfScopePrecomputer<'a, 'tcx> {295    visited: DenseBitSet<mir::BasicBlock>,296    visit_stack: Vec<mir::BasicBlock>,297    body: &'a Body<'tcx>,298    regioncx: &'a RegionInferenceContext<'tcx>,299300    loans_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,301}302303impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {304    fn compute(305        body: &Body<'tcx>,306        regioncx: &RegionInferenceContext<'tcx>,307        borrow_set: &BorrowSet<'tcx>,308    ) -> FxIndexMap<Location, Vec<BorrowIndex>> {309        // The in-tree polonius analysis computes loans going out of scope using the310        // set-of-loans model.311        let mut prec = PoloniusOutOfScopePrecomputer {312            visited: DenseBitSet::new_empty(body.basic_blocks.len()),313            visit_stack: vec![],314            body,315            regioncx,316            loans_out_of_scope_at_location: FxIndexMap::default(),317        };318        for (loan_idx, loan_data) in borrow_set.iter_enumerated() {319            let loan_issued_at = loan_data.reserve_location;320            prec.precompute_loans_out_of_scope(loan_idx, loan_issued_at);321        }322323        prec.loans_out_of_scope_at_location324    }325326    /// Loans are in scope while they are live: whether they are contained within any live region.327    /// In the location-insensitive analysis, a loan will be contained in a region if the issuing328    /// region can reach it in the subset graph. So this is a reachability problem.329    fn precompute_loans_out_of_scope(&mut self, loan_idx: BorrowIndex, loan_issued_at: Location) {330        let first_block = loan_issued_at.block;331        let first_bb_data = &self.body.basic_blocks[first_block];332333        // The first block we visit is the one where the loan is issued, starting from the statement334        // where the loan is issued: at `loan_issued_at`.335        let first_lo = loan_issued_at.statement_index;336        let first_hi = first_bb_data.statements.len();337338        if let Some(kill_location) =339            self.loan_kill_location(loan_idx, loan_issued_at, first_block, first_lo, first_hi)340        {341            debug!("loan {:?} gets killed at {:?}", loan_idx, kill_location);342            self.loans_out_of_scope_at_location.entry(kill_location).or_default().push(loan_idx);343344            // The loan dies within the first block, we're done and can early return.345            return;346        }347348        // The loan is not dead. Add successor BBs to the work list, if necessary.349        for succ_bb in first_bb_data.terminator().successors() {350            if self.visited.insert(succ_bb) {351                self.visit_stack.push(succ_bb);352            }353        }354355        // We may end up visiting `first_block` again. This is not an issue: we know at this point356        // that the loan is not killed in the `first_lo..=first_hi` range, so checking the357        // `0..first_lo` range and the `0..first_hi` range gives the same result.358        while let Some(block) = self.visit_stack.pop() {359            let bb_data = &self.body[block];360            let num_stmts = bb_data.statements.len();361            if let Some(kill_location) =362                self.loan_kill_location(loan_idx, loan_issued_at, block, 0, num_stmts)363            {364                debug!("loan {:?} gets killed at {:?}", loan_idx, kill_location);365                self.loans_out_of_scope_at_location366                    .entry(kill_location)367                    .or_default()368                    .push(loan_idx);369370                // The loan dies within this block, so we don't need to visit its successors.371                continue;372            }373374            // Add successor BBs to the work list, if necessary.375            for succ_bb in bb_data.terminator().successors() {376                if self.visited.insert(succ_bb) {377                    self.visit_stack.push(succ_bb);378                }379            }380        }381382        self.visited.clear();383        assert!(self.visit_stack.is_empty(), "visit stack should be empty");384    }385386    /// Returns the lowest statement in `start..=end`, where the loan goes out of scope, if any.387    /// This is the statement where the issuing region can't reach any of the regions that are live388    /// at this point.389    fn loan_kill_location(390        &self,391        loan_idx: BorrowIndex,392        loan_issued_at: Location,393        block: BasicBlock,394        start: usize,395        end: usize,396    ) -> Option<Location> {397        for statement_index in start..=end {398            let location = Location { block, statement_index };399400            // Check whether the issuing region can reach local regions that are live at this point:401            // - a loan is always live at its issuing location because it can reach the issuing402            // region, which is always live at this location.403            if location == loan_issued_at {404                continue;405            }406407            // - the loan goes out of scope at `location` if it's not contained within any regions408            // live at this point.409            //410            // FIXME: if the issuing region `i` can reach a live region `r` at point `p`, and `r` is411            // live at point `q`, then it's guaranteed that `i` would reach `r` at point `q`.412            // Reachability is location-insensitive, and we could take advantage of that, by jumping413            // to a further point than just the next statement: we can jump to the furthest point414            // within the block where `r` is live.415            if self.regioncx.is_loan_live_at(loan_idx, location) {416                continue;417            }418419            // No live region is reachable from the issuing region: the loan is killed at this420            // point.421            return Some(location);422        }423424        None425    }426}427428impl<'a, 'tcx> Borrows<'a, 'tcx> {429    pub fn new(430        tcx: TyCtxt<'tcx>,431        body: &'a Body<'tcx>,432        regioncx: &RegionInferenceContext<'tcx>,433        borrow_set: &'a BorrowSet<'tcx>,434    ) -> Self {435        let borrows_out_of_scope_at_location =436            if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {437                calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set)438            } else {439                PoloniusOutOfScopePrecomputer::compute(body, regioncx, borrow_set)440            };441        Borrows { tcx, body, borrow_set, borrows_out_of_scope_at_location }442    }443444    /// Add all borrows to the kill set, if those borrows are out of scope at `location`.445    /// That means they went out of a nonlexical scope446    fn kill_loans_out_of_scope_at_location(447        &self,448        state: &mut <Self as Analysis<'tcx>>::Domain,449        location: Location,450    ) {451        // NOTE: The state associated with a given `location`452        // reflects the dataflow on entry to the statement.453        // Iterate over each of the borrows that we've precomputed454        // to have went out of scope at this location and kill them.455        //456        // We are careful always to call this function *before* we457        // set up the gen-bits for the statement or458        // terminator. That way, if the effect of the statement or459        // terminator *does* introduce a new loan of the same460        // region, then setting that gen-bit will override any461        // potential kill introduced here.462        if let Some(indices) = self.borrows_out_of_scope_at_location.get(&location) {463            state.kill_all(indices.iter().copied());464        }465    }466467    /// Kill any borrows that conflict with `place`.468    fn kill_borrows_on_place(469        &self,470        state: &mut <Self as Analysis<'tcx>>::Domain,471        place: Place<'tcx>,472    ) {473        debug!("kill_borrows_on_place: place={:?}", place);474475        let other_borrows_of_local = self476            .borrow_set477            .local_map478            .get(&place.local)479            .into_iter()480            .flat_map(|bs| bs.iter())481            .copied();482483        // If the borrowed place is a local with no projections, all other borrows of this484        // local must conflict. This is purely an optimization so we don't have to call485        // `places_conflict` for every borrow.486        if place.projection.is_empty() {487            if !self.body.local_decls[place.local].is_ref_to_static() {488                state.kill_all(other_borrows_of_local);489            }490            return;491        }492493        // By passing `PlaceConflictBias::NoOverlap`, we conservatively assume that any given494        // pair of array indices are not equal, so that when `places_conflict` returns true, we495        // will be assured that two places being compared definitely denotes the same sets of496        // locations.497        let definitely_conflicting_borrows = other_borrows_of_local.filter(|&i| {498            places_conflict(499                self.tcx,500                self.body,501                self.borrow_set[i].borrowed_place,502                place,503                PlaceConflictBias::NoOverlap,504            )505        });506507        state.kill_all(definitely_conflicting_borrows);508    }509}510511type BorrowsDomain = MixedBitSet<BorrowIndex>;512513/// Forward dataflow computation of the set of borrows that are in scope at a particular location.514/// - we gen the introduced loans515/// - we kill loans on locals going out of (regular) scope516/// - we kill the loans going out of their region's NLL scope: in NLL terms, the frontier where a517///   region stops containing the CFG points reachable from the issuing location.518/// - we also kill loans of conflicting places when overwriting a shared path: e.g. borrows of519///   `a.b.c` when `a` is overwritten.520impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> {521    type Domain = BorrowsDomain;522523    const NAME: &'static str = "borrows";524525    fn bottom_value(&self, _: &mir::Body<'tcx>) -> Self::Domain {526        // bottom = nothing is reserved or activated yet;527        MixedBitSet::new_empty(self.borrow_set.len())528    }529530    fn initialize_start_block(&self, _: &mir::Body<'tcx>, _: &mut Self::Domain) {531        // no borrows of code region_scopes have been taken prior to532        // function execution, so this method has no effect.533    }534535    fn apply_early_statement_effect(536        &self,537        state: &mut Self::Domain,538        _statement: &mir::Statement<'tcx>,539        location: Location,540    ) {541        self.kill_loans_out_of_scope_at_location(state, location);542    }543544    fn apply_primary_statement_effect(545        &self,546        state: &mut Self::Domain,547        stmt: &mir::Statement<'tcx>,548        location: Location,549    ) {550        match &stmt.kind {551            mir::StatementKind::Assign(box (lhs, rhs)) => {552                if let mir::Rvalue::Ref(_, _, place) = rhs {553                    if place.ignore_borrow(554                        self.tcx,555                        self.body,556                        &self.borrow_set.locals_state_at_exit,557                    ) {558                        return;559                    }560                    let index = self.borrow_set.get_index_of(&location).unwrap_or_else(|| {561                        panic!("could not find BorrowIndex for location {location:?}");562                    });563564                    state.gen_(index);565                }566567                // Make sure there are no remaining borrows for variables568                // that are assigned over.569                self.kill_borrows_on_place(state, *lhs);570            }571572            mir::StatementKind::StorageDead(local) => {573                // Make sure there are no remaining borrows for locals that574                // are gone out of scope.575                self.kill_borrows_on_place(state, Place::from(*local));576            }577578            mir::StatementKind::FakeRead(..)579            | mir::StatementKind::SetDiscriminant { .. }580            | mir::StatementKind::StorageLive(..)581            | mir::StatementKind::PlaceMention(..)582            | mir::StatementKind::AscribeUserType(..)583            | mir::StatementKind::Coverage(..)584            | mir::StatementKind::Intrinsic(..)585            | mir::StatementKind::ConstEvalCounter586            | mir::StatementKind::BackwardIncompatibleDropHint { .. }587            | mir::StatementKind::Nop => {}588        }589    }590591    fn apply_early_terminator_effect(592        &self,593        state: &mut Self::Domain,594        _terminator: &mir::Terminator<'tcx>,595        location: Location,596    ) {597        self.kill_loans_out_of_scope_at_location(state, location);598    }599600    fn apply_primary_terminator_effect<'mir>(601        &self,602        state: &mut Self::Domain,603        terminator: &'mir mir::Terminator<'tcx>,604        _location: Location,605    ) -> TerminatorEdges<'mir, 'tcx> {606        if let mir::TerminatorKind::InlineAsm { operands, .. } = &terminator.kind {607            for op in operands {608                if let mir::InlineAsmOperand::Out { place: Some(place), .. }609                | mir::InlineAsmOperand::InOut { out_place: Some(place), .. } = *op610                {611                    self.kill_borrows_on_place(state, place);612                }613            }614        }615        terminator.edges()616    }617}618619impl<C> DebugWithContext<C> for BorrowIndex {}