1use std::cmp;2use std::collections::BTreeSet;3use std::sync::Arc;4use std::time::{Duration, Instant};56use itertools::Itertools;7use rustc_abi::FIRST_VARIANT;8use rustc_ast::expand::allocator::{9    ALLOC_ERROR_HANDLER, ALLOCATOR_METHODS, AllocatorKind, AllocatorMethod, AllocatorMethodInput,10    AllocatorTy,11};12use rustc_data_structures::fx::{FxHashMap, FxIndexSet};13use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};14use rustc_data_structures::sync::{IntoDynSyncSend, par_map};15use rustc_data_structures::unord::UnordMap;16use rustc_hir::attrs::{DebuggerVisualizerType, OptimizeAttr};17use rustc_hir::def_id::{DefId, LOCAL_CRATE};18use rustc_hir::lang_items::LangItem;19use rustc_hir::{ItemId, Target, find_attr};20use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;21use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;22use rustc_middle::middle::dependency_format::Dependencies;23use rustc_middle::middle::exported_symbols::{self, SymbolExportKind};24use rustc_middle::middle::lang_items;25use rustc_middle::mir::BinOp;26use rustc_middle::mir::interpret::ErrorHandled;27use rustc_middle::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem, MonoItemPartitions};28use rustc_middle::query::Providers;29use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};30use rustc_middle::ty::{self, Instance, PatternKind, Ty, TyCtxt, Unnormalized};31use rustc_middle::{bug, span_bug};32use rustc_session::Session;33use rustc_session::config::{self, CrateType, EntryFnType};34use rustc_span::{DUMMY_SP, Symbol};35use rustc_symbol_mangling::mangle_internal_symbol;36use rustc_target::spec::{Arch, Os};37use rustc_trait_selection::infer::{BoundRegionConversionTime, TyCtxtInferExt};38use rustc_trait_selection::traits::{ObligationCause, ObligationCtxt};39use tracing::{debug, info};4041use crate::assert_module_sources::CguReuse;42use crate::back::link::are_upstream_rust_objects_already_included;43use crate::back::write::{44    ComputedLtoType, OngoingCodegen, compute_per_cgu_lto_type, start_async_codegen,45    submit_codegened_module_to_llvm, submit_post_lto_module_to_llvm, submit_pre_lto_module_to_llvm,46};47use crate::common::{self, IntPredicate, RealPredicate, TypeKind};48use crate::meth::load_vtable;49use crate::mir::operand::OperandValue;50use crate::mir::place::PlaceRef;51use crate::traits::*;52use crate::{CachedModuleCodegen, CodegenLintLevels, CrateInfo, ModuleCodegen, errors, meth, mir};5354pub(crate) fn bin_op_to_icmp_predicate(op: BinOp, signed: bool) -> IntPredicate {55    match (op, signed) {56        (BinOp::Eq, _) => IntPredicate::IntEQ,57        (BinOp::Ne, _) => IntPredicate::IntNE,58        (BinOp::Lt, true) => IntPredicate::IntSLT,59        (BinOp::Lt, false) => IntPredicate::IntULT,60        (BinOp::Le, true) => IntPredicate::IntSLE,61        (BinOp::Le, false) => IntPredicate::IntULE,62        (BinOp::Gt, true) => IntPredicate::IntSGT,63        (BinOp::Gt, false) => IntPredicate::IntUGT,64        (BinOp::Ge, true) => IntPredicate::IntSGE,65        (BinOp::Ge, false) => IntPredicate::IntUGE,66        op => bug!("bin_op_to_icmp_predicate: expected comparison operator, found {:?}", op),67    }68}6970pub(crate) fn bin_op_to_fcmp_predicate(op: BinOp) -> RealPredicate {71    match op {72        BinOp::Eq => RealPredicate::RealOEQ,73        BinOp::Ne => RealPredicate::RealUNE,74        BinOp::Lt => RealPredicate::RealOLT,75        BinOp::Le => RealPredicate::RealOLE,76        BinOp::Gt => RealPredicate::RealOGT,77        BinOp::Ge => RealPredicate::RealOGE,78        op => bug!("bin_op_to_fcmp_predicate: expected comparison operator, found {:?}", op),79    }80}8182pub fn compare_simd_types<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(83    bx: &mut Bx,84    lhs: Bx::Value,85    rhs: Bx::Value,86    t: Ty<'tcx>,87    ret_ty: Bx::Type,88    op: BinOp,89) -> Bx::Value {90    let signed = match t.kind() {91        ty::Float(_) => {92            let cmp = bin_op_to_fcmp_predicate(op);93            let cmp = bx.fcmp(cmp, lhs, rhs);94            return bx.sext(cmp, ret_ty);95        }96        ty::Uint(_) => false,97        ty::Int(_) => true,98        _ => bug!("compare_simd_types: invalid SIMD type"),99    };100101    let cmp = bin_op_to_icmp_predicate(op, signed);102    let cmp = bx.icmp(cmp, lhs, rhs);103    // LLVM outputs an `< size x i1 >`, so we need to perform a sign extension104    // to get the correctly sized type. This will compile to a single instruction105    // once the IR is converted to assembly if the SIMD instruction is supported106    // by the target architecture.107    bx.sext(cmp, ret_ty)108}109110/// Codegen takes advantage of the additional assumption, where if the111/// principal trait def id of what's being casted doesn't change,112/// then we don't need to adjust the vtable at all. This113/// corresponds to the fact that `dyn Tr<A>: Unsize<dyn Tr<B>>`114/// requires that `A = B`; we don't allow *upcasting* objects115/// between the same trait with different args. If we, for116/// some reason, were to relax the `Unsize` trait, it could become117/// unsound, so let's validate here that the trait refs are subtypes.118pub fn validate_trivial_unsize<'tcx>(119    tcx: TyCtxt<'tcx>,120    source_data: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,121    target_data: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,122) -> bool {123    match (source_data.principal(), target_data.principal()) {124        (Some(hr_source_principal), Some(hr_target_principal)) => {125            let (infcx, param_env) =126                tcx.infer_ctxt().build_with_typing_env(ty::TypingEnv::fully_monomorphized());127            let universe = infcx.universe();128            let ocx = ObligationCtxt::new(&infcx);129            infcx.enter_forall(hr_target_principal, |target_principal| {130                let source_principal = infcx.instantiate_binder_with_fresh_vars(131                    DUMMY_SP,132                    BoundRegionConversionTime::HigherRankedType,133                    hr_source_principal,134                );135                let Ok(()) = ocx.eq(136                    &ObligationCause::dummy(),137                    param_env,138                    target_principal,139                    source_principal,140                ) else {141                    return false;142                };143                if !ocx.evaluate_obligations_error_on_ambiguity().is_empty() {144                    return false;145                }146                infcx.leak_check(universe, None).is_ok()147            })148        }149        (_, None) => true,150        _ => false,151    }152}153154/// Retrieves the information we are losing (making dynamic) in an unsizing155/// adjustment.156///157/// The `old_info` argument is a bit odd. It is intended for use in an upcast,158/// where the new vtable for an object will be derived from the old one.159fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(160    bx: &mut Bx,161    source: Ty<'tcx>,162    target: Ty<'tcx>,163    old_info: Option<Bx::Value>,164) -> Bx::Value {165    let cx = bx.cx();166    let (source, target) =167        cx.tcx().struct_lockstep_tails_for_codegen(source, target, bx.typing_env());168    match (source.kind(), target.kind()) {169        (&ty::Array(_, len), &ty::Slice(_)) => cx.const_usize(170            len.try_to_target_usize(cx.tcx()).expect("expected monomorphic const in codegen"),171        ),172        (&ty::Dynamic(data_a, _), &ty::Dynamic(data_b, _)) => {173            let old_info =174                old_info.expect("unsized_info: missing old info for trait upcasting coercion");175            let b_principal_def_id = data_b.principal_def_id();176            if data_a.principal_def_id() == b_principal_def_id || b_principal_def_id.is_none() {177                // Codegen takes advantage of the additional assumption, where if the178                // principal trait def id of what's being casted doesn't change,179                // then we don't need to adjust the vtable at all. This180                // corresponds to the fact that `dyn Tr<A>: Unsize<dyn Tr<B>>`181                // requires that `A = B`; we don't allow *upcasting* objects182                // between the same trait with different args. If we, for183                // some reason, were to relax the `Unsize` trait, it could become184                // unsound, so let's assert here that the trait refs are *equal*.185                debug_assert!(186                    validate_trivial_unsize(cx.tcx(), data_a, data_b),187                    "NOP unsize vtable changed principal trait ref: {data_a} -> {data_b}"188                );189190                // A NOP cast that doesn't actually change anything, let's avoid any191                // unnecessary work. This relies on the assumption that if the principal192                // traits are equal, then the associated type bounds (`dyn Trait<Assoc=T>`)193                // are also equal, which is ensured by the fact that normalization is194                // a function and we do not allow overlapping impls.195                return old_info;196            }197198            // trait upcasting coercion199200            let vptr_entry_idx = cx.tcx().supertrait_vtable_slot((source, target));201202            if let Some(entry_idx) = vptr_entry_idx {203                let ptr_size = bx.data_layout().pointer_size();204                let vtable_byte_offset = u64::try_from(entry_idx).unwrap() * ptr_size.bytes();205                load_vtable(bx, old_info, bx.type_ptr(), vtable_byte_offset, source, true)206            } else {207                old_info208            }209        }210        (_, ty::Dynamic(data, _)) => meth::get_vtable(211            cx,212            source,213            data.principal()214                .map(|principal| bx.tcx().instantiate_bound_regions_with_erased(principal)),215        ),216        _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),217    }218}219220/// Coerces `src` to `dst_ty`. `src_ty` must be a pointer.221pub(crate) fn unsize_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(222    bx: &mut Bx,223    src: Bx::Value,224    src_ty: Ty<'tcx>,225    dst_ty: Ty<'tcx>,226    old_info: Option<Bx::Value>,227) -> (Bx::Value, Bx::Value) {228    debug!("unsize_ptr: {:?} => {:?}", src_ty, dst_ty);229    match (src_ty.kind(), dst_ty.kind()) {230        (&ty::Pat(a, _), &ty::Pat(b, _)) => unsize_ptr(bx, src, a, b, old_info),231        (&ty::Ref(_, a, _), &ty::Ref(_, b, _) | &ty::RawPtr(b, _))232        | (&ty::RawPtr(a, _), &ty::RawPtr(b, _)) => {233            assert_eq!(bx.cx().type_is_sized(a), old_info.is_none());234            (src, unsized_info(bx, a, b, old_info))235        }236        (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {237            assert_eq!(def_a, def_b); // implies same number of fields238            let src_layout = bx.cx().layout_of(src_ty);239            let dst_layout = bx.cx().layout_of(dst_ty);240            if src_ty == dst_ty {241                return (src, old_info.unwrap());242            }243            let mut result = None;244            for i in 0..src_layout.fields.count() {245                let src_f = src_layout.field(bx.cx(), i);246                if src_f.is_1zst() {247                    // We are looking for the one non-1-ZST field; this is not it.248                    continue;249                }250251                assert_eq!(src_layout.fields.offset(i).bytes(), 0);252                assert_eq!(dst_layout.fields.offset(i).bytes(), 0);253                assert_eq!(src_layout.size, src_f.size);254255                let dst_f = dst_layout.field(bx.cx(), i);256                assert_ne!(src_f.ty, dst_f.ty);257                assert_eq!(result, None);258                result = Some(unsize_ptr(bx, src, src_f.ty, dst_f.ty, old_info));259            }260            result.unwrap()261        }262        _ => bug!("unsize_ptr: called on bad types"),263    }264}265266/// Coerces `src`, which is a reference to a value of type `src_ty`,267/// to a value of type `dst_ty`, and stores the result in `dst`.268pub(crate) fn coerce_unsized_into<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(269    bx: &mut Bx,270    src: PlaceRef<'tcx, Bx::Value>,271    dst: PlaceRef<'tcx, Bx::Value>,272) {273    let src_ty = src.layout.ty;274    let dst_ty = dst.layout.ty;275    match (src_ty.kind(), dst_ty.kind()) {276        (&ty::Pat(s, sp), &ty::Pat(d, dp))277            if let (PatternKind::NotNull, PatternKind::NotNull) = (*sp, *dp) =>278        {279            let src = src.project_type(bx, s);280            let dst = dst.project_type(bx, d);281            coerce_unsized_into(bx, src, dst)282        }283        (&ty::Ref(..), &ty::Ref(..) | &ty::RawPtr(..)) | (&ty::RawPtr(..), &ty::RawPtr(..)) => {284            let (base, info) = match bx.load_operand(src).val {285                OperandValue::Pair(base, info) => unsize_ptr(bx, base, src_ty, dst_ty, Some(info)),286                OperandValue::Immediate(base) => unsize_ptr(bx, base, src_ty, dst_ty, None),287                OperandValue::Ref(..) | OperandValue::ZeroSized => bug!(),288            };289            OperandValue::Pair(base, info).store(bx, dst);290        }291292        (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {293            assert_eq!(def_a, def_b); // implies same number of fields294295            for i in def_a.variant(FIRST_VARIANT).fields.indices() {296                let src_f = src.project_field(bx, i.as_usize());297                let dst_f = dst.project_field(bx, i.as_usize());298299                if dst_f.layout.is_zst() {300                    // No data here, nothing to copy/coerce.301                    continue;302                }303304                if src_f.layout.ty == dst_f.layout.ty {305                    bx.typed_place_copy(dst_f.val, src_f.val, src_f.layout);306                } else {307                    coerce_unsized_into(bx, src_f, dst_f);308                }309            }310        }311        _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty,),312    }313}314315/// Returns `rhs` sufficiently masked, truncated, and/or extended so that it can be used to shift316/// `lhs`: it has the same size as `lhs`, and the value, when interpreted unsigned (no matter its317/// type), will not exceed the size of `lhs`.318///319/// Shifts in MIR are all allowed to have mismatched LHS & RHS types, and signed RHS.320/// The shift methods in `BuilderMethods`, however, are fully homogeneous321/// (both parameters and the return type are all the same size) and assume an unsigned RHS.322///323/// If `is_unchecked` is false, this masks the RHS to ensure it stays in-bounds,324/// as the `BuilderMethods` shifts are UB for out-of-bounds shift amounts.325/// For 32- and 64-bit types, this matches the semantics326/// of Java. (See related discussion on #1877 and #10183.)327///328/// If `is_unchecked` is true, this does no masking, and adds sufficient `assume`329/// calls or operation flags to preserve as much freedom to optimize as possible.330pub(crate) fn build_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(331    bx: &mut Bx,332    lhs: Bx::Value,333    mut rhs: Bx::Value,334    is_unchecked: bool,335) -> Bx::Value {336    // Shifts may have any size int on the rhs337    let mut rhs_llty = bx.cx().val_ty(rhs);338    let mut lhs_llty = bx.cx().val_ty(lhs);339340    let mask = common::shift_mask_val(bx, lhs_llty, rhs_llty, false);341    if !is_unchecked {342        rhs = bx.and(rhs, mask);343    }344345    if bx.cx().type_kind(rhs_llty) == TypeKind::Vector {346        rhs_llty = bx.cx().element_type(rhs_llty)347    }348    if bx.cx().type_kind(lhs_llty) == TypeKind::Vector {349        lhs_llty = bx.cx().element_type(lhs_llty)350    }351    let rhs_sz = bx.cx().int_width(rhs_llty);352    let lhs_sz = bx.cx().int_width(lhs_llty);353    if lhs_sz < rhs_sz {354        if is_unchecked { bx.unchecked_utrunc(rhs, lhs_llty) } else { bx.trunc(rhs, lhs_llty) }355    } else if lhs_sz > rhs_sz {356        // We zero-extend even if the RHS is signed. So e.g. `(x: i32) << -1i8` will zero-extend the357        // RHS to `255i32`. But then we mask the shift amount to be within the size of the LHS358        // anyway so the result is `31` as it should be. All the extra bits introduced by zext359        // are masked off so their value does not matter.360        // FIXME: if we ever support 512bit integers, this will be wrong! For such large integers,361        // the extra bits introduced by zext are *not* all masked away any more.362        assert!(lhs_sz <= 256);363        bx.zext(rhs, lhs_llty)364    } else {365        rhs366    }367}368369// Returns `true` if this session's target will use native wasm370// exceptions. This means that the VM does the unwinding for371// us372pub fn wants_wasm_eh(sess: &Session) -> bool {373    sess.target.is_like_wasm374        && (sess.target.os != Os::Emscripten || sess.opts.unstable_opts.emscripten_wasm_eh)375}376377/// Returns `true` if this session's target will use SEH-based unwinding.378///379/// This is only true for MSVC targets, and even then the 64-bit MSVC target380/// currently uses SEH-ish unwinding with DWARF info tables to the side (same as381/// 64-bit MinGW) instead of "full SEH".382pub fn wants_msvc_seh(sess: &Session) -> bool {383    sess.target.is_like_msvc384}385386/// Returns `true` if this session's target requires the new exception387/// handling LLVM IR instructions (catchpad / cleanuppad / ... instead388/// of landingpad)389pub(crate) fn wants_new_eh_instructions(sess: &Session) -> bool {390    wants_wasm_eh(sess) || wants_msvc_seh(sess)391}392393pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(394    cx: &'a Bx::CodegenCx,395    instance: Instance<'tcx>,396) {397    // this is an info! to allow collecting monomorphization statistics398    // and to allow finding the last function before LLVM aborts from399    // release builds.400    info!("codegen_instance({})", instance);401402    mir::codegen_mir::<Bx>(cx, instance);403}404405pub fn codegen_global_asm<'tcx, Cx>(cx: &mut Cx, item_id: ItemId)406where407    Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> + AsmCodegenMethods<'tcx>,408{409    let item = cx.tcx().hir_item(item_id);410    if let rustc_hir::ItemKind::GlobalAsm { asm, .. } = item.kind {411        let operands: Vec<_> = asm412            .operands413            .iter()414            .map(|(op, op_sp)| match *op {415                rustc_hir::InlineAsmOperand::Const { ref anon_const } => {416                    match cx.tcx().const_eval_poly(anon_const.def_id.to_def_id()) {417                        Ok(const_value) => {418                            let ty =419                                cx.tcx().typeck_body(anon_const.body).node_type(anon_const.hir_id);420                            let string = common::asm_const_to_str(421                                cx.tcx(),422                                *op_sp,423                                const_value,424                                cx.layout_of(ty),425                            );426                            GlobalAsmOperandRef::Const { string }427                        }428                        Err(ErrorHandled::Reported { .. }) => {429                            // An error has already been reported and430                            // compilation is guaranteed to fail if execution431                            // hits this path. So an empty string instead of432                            // a stringified constant value will suffice.433                            GlobalAsmOperandRef::Const { string: String::new() }434                        }435                        Err(ErrorHandled::TooGeneric(_)) => {436                            span_bug!(*op_sp, "asm const cannot be resolved; too generic")437                        }438                    }439                }440                rustc_hir::InlineAsmOperand::SymFn { expr } => {441                    let ty = cx.tcx().typeck(item_id.owner_id).expr_ty(expr);442                    let instance = match ty.kind() {443                        &ty::FnDef(def_id, args) => Instance::expect_resolve(444                            cx.tcx(),445                            ty::TypingEnv::fully_monomorphized(),446                            def_id,447                            args,448                            expr.span,449                        ),450                        _ => span_bug!(*op_sp, "asm sym is not a function"),451                    };452453                    GlobalAsmOperandRef::SymFn { instance }454                }455                rustc_hir::InlineAsmOperand::SymStatic { path: _, def_id } => {456                    GlobalAsmOperandRef::SymStatic { def_id }457                }458                rustc_hir::InlineAsmOperand::In { .. }459                | rustc_hir::InlineAsmOperand::Out { .. }460                | rustc_hir::InlineAsmOperand::InOut { .. }461                | rustc_hir::InlineAsmOperand::SplitInOut { .. }462                | rustc_hir::InlineAsmOperand::Label { .. } => {463                    span_bug!(*op_sp, "invalid operand type for global_asm!")464                }465            })466            .collect();467468        cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);469    } else {470        span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")471    }472}473474/// Creates the `main` function which will initialize the rust runtime and call475/// users main function.476pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(477    cx: &'a Bx::CodegenCx,478    cgu: &CodegenUnit<'tcx>,479) -> Option<Bx::Function> {480    let (main_def_id, entry_type) = cx.tcx().entry_fn(())?;481    let main_is_local = main_def_id.is_local();482    let instance = Instance::mono(cx.tcx(), main_def_id);483484    if main_is_local {485        // We want to create the wrapper in the same codegen unit as Rust's main486        // function.487        if !cgu.contains_item(&MonoItem::Fn(instance)) {488            return None;489        }490    } else if !cgu.is_primary() {491        // We want to create the wrapper only when the codegen unit is the primary one492        return None;493    }494495    let main_llfn = cx.get_fn_addr(instance);496497    let entry_fn = create_entry_fn::<Bx>(cx, main_llfn, main_def_id, entry_type);498    return Some(entry_fn);499500    fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(501        cx: &'a Bx::CodegenCx,502        rust_main: Bx::Value,503        rust_main_def_id: DefId,504        entry_type: EntryFnType,505    ) -> Bx::Function {506        // The entry function is either `int main(void)` or `int main(int argc, char **argv)`, or507        // `usize efi_main(void *handle, void *system_table)` depending on the target.508        let llfty = if cx.sess().target.os == Os::Uefi {509            cx.type_func(&[cx.type_ptr(), cx.type_ptr()], cx.type_isize())510        } else if cx.sess().target.main_needs_argc_argv {511            cx.type_func(&[cx.type_int(), cx.type_ptr()], cx.type_int())512        } else {513            cx.type_func(&[], cx.type_int())514        };515516        let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).no_bound_vars().unwrap().output();517        // Given that `main()` has no arguments,518        // then its return type cannot have519        // late-bound regions, since late-bound520        // regions must appear in the argument521        // listing.522        let main_ret_ty = cx.tcx().normalize_erasing_regions(523            cx.typing_env(),524            Unnormalized::new_wip(main_ret_ty.no_bound_vars().unwrap()),525        );526527        let Some(llfn) = cx.declare_c_main(llfty) else {528            // FIXME: We should be smart and show a better diagnostic here.529            let span = cx.tcx().def_span(rust_main_def_id);530            cx.tcx().dcx().emit_fatal(errors::MultipleMainFunctions { span });531        };532533        // `main` should respect same config for frame pointer elimination as rest of code534        cx.set_frame_pointer_type(llfn);535        cx.apply_target_cpu_attr(llfn);536537        let llbb = Bx::append_block(cx, llfn, "top");538        let mut bx = Bx::build(cx, llbb);539540        bx.insert_reference_to_gdb_debug_scripts_section_global();541542        let isize_ty = cx.type_isize();543        let ptr_ty = cx.type_ptr();544        let (arg_argc, arg_argv) = get_argc_argv(&mut bx);545546        let EntryFnType::Main { sigpipe } = entry_type;547        let (start_fn, start_ty, args, instance) = {548            let start_def_id = cx.tcx().require_lang_item(LangItem::Start, DUMMY_SP);549            let start_instance = ty::Instance::expect_resolve(550                cx.tcx(),551                cx.typing_env(),552                start_def_id,553                cx.tcx().mk_args(&[main_ret_ty.into()]),554                DUMMY_SP,555            );556            let start_fn = cx.get_fn_addr(start_instance);557558            let i8_ty = cx.type_i8();559            let arg_sigpipe = bx.const_u8(sigpipe);560561            let start_ty = cx.type_func(&[cx.val_ty(rust_main), isize_ty, ptr_ty, i8_ty], isize_ty);562            (563                start_fn,564                start_ty,565                vec![rust_main, arg_argc, arg_argv, arg_sigpipe],566                Some(start_instance),567            )568        };569570        let result = bx.call(start_ty, None, None, start_fn, &args, None, instance);571        if cx.sess().target.os == Os::Uefi {572            bx.ret(result);573        } else {574            let cast = bx.intcast(result, cx.type_int(), true);575            bx.ret(cast);576        }577578        llfn579    }580}581582/// Obtain the `argc` and `argv` values to pass to the rust start function583/// (i.e., the "start" lang item).584fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(bx: &mut Bx) -> (Bx::Value, Bx::Value) {585    if bx.cx().sess().target.os == Os::Uefi {586        // Params for UEFI587        let param_handle = bx.get_param(0);588        let param_system_table = bx.get_param(1);589        let ptr_size = bx.tcx().data_layout.pointer_size();590        let ptr_align = bx.tcx().data_layout.pointer_align().abi;591        let arg_argc = bx.const_int(bx.cx().type_isize(), 2);592        let arg_argv = bx.alloca(2 * ptr_size, ptr_align);593        bx.store(param_handle, arg_argv, ptr_align);594        let arg_argv_el1 = bx.inbounds_ptradd(arg_argv, bx.const_usize(ptr_size.bytes()));595        bx.store(param_system_table, arg_argv_el1, ptr_align);596        (arg_argc, arg_argv)597    } else if bx.cx().sess().target.main_needs_argc_argv {598        // Params from native `main()` used as args for rust start function599        let param_argc = bx.get_param(0);600        let param_argv = bx.get_param(1);601        let arg_argc = bx.intcast(param_argc, bx.cx().type_isize(), true);602        let arg_argv = param_argv;603        (arg_argc, arg_argv)604    } else {605        // The Rust start function doesn't need `argc` and `argv`, so just pass zeros.606        let arg_argc = bx.const_int(bx.cx().type_int(), 0);607        let arg_argv = bx.const_null(bx.cx().type_ptr());608        (arg_argc, arg_argv)609    }610}611612/// This function returns all of the debugger visualizers specified for the613/// current crate as well as all upstream crates transitively that match the614/// `visualizer_type` specified.615pub fn collect_debugger_visualizers_transitive(616    tcx: TyCtxt<'_>,617    visualizer_type: DebuggerVisualizerType,618) -> BTreeSet<DebuggerVisualizerFile> {619    tcx.debugger_visualizers(LOCAL_CRATE)620        .iter()621        .chain(622            tcx.crates(())623                .iter()624                .filter(|&cnum| {625                    let used_crate_source = tcx.used_crate_source(*cnum);626                    used_crate_source.rlib.is_some() || used_crate_source.rmeta.is_some()627                })628                .flat_map(|&cnum| tcx.debugger_visualizers(cnum)),629        )630        .filter(|visualizer| visualizer.visualizer_type == visualizer_type)631        .cloned()632        .collect::<BTreeSet<_>>()633}634635/// Decide allocator kind to codegen. If `Some(_)` this will be the same as636/// `tcx.allocator_kind`, but it may be `None` in more cases (e.g. if using637/// allocator definitions from a dylib dependency).638pub fn allocator_kind_for_codegen(tcx: TyCtxt<'_>) -> Option<AllocatorKind> {639    // If the crate doesn't have an `allocator_kind` set then there's definitely640    // no shim to generate. Otherwise we also check our dependency graph for all641    // our output crate types. If anything there looks like its a `Dynamic`642    // linkage for all crate types we may link as, then it's already got an643    // allocator shim and we'll be using that one instead. If nothing exists644    // then it's our job to generate the allocator! If crate types disagree645    // about whether an allocator shim is necessary or not, we generate one646    // and let needs_allocator_shim_for_linking decide at link time whether or647    // not to use it for any particular linker invocation.648    let all_crate_types_any_dynamic_crate = tcx.dependency_formats(()).iter().all(|(_, list)| {649        use rustc_middle::middle::dependency_format::Linkage;650        list.iter().any(|&linkage| linkage == Linkage::Dynamic)651    });652    if all_crate_types_any_dynamic_crate { None } else { tcx.allocator_kind(()) }653}654655/// Decide if this particular crate type needs an allocator shim linked in.656/// This may return true even when allocator_kind_for_codegen returns false. In657/// this case no allocator shim shall be linked.658pub(crate) fn needs_allocator_shim_for_linking(659    dependency_formats: &Dependencies,660    crate_type: CrateType,661) -> bool {662    use rustc_middle::middle::dependency_format::Linkage;663    let any_dynamic_crate =664        dependency_formats[&crate_type].iter().any(|&linkage| linkage == Linkage::Dynamic);665    !any_dynamic_crate666}667668pub fn allocator_shim_contents(tcx: TyCtxt<'_>, kind: AllocatorKind) -> Vec<AllocatorMethod> {669    let mut methods = Vec::new();670671    if kind == AllocatorKind::Default {672        methods.extend(ALLOCATOR_METHODS.into_iter().copied());673    }674675    // If the return value of allocator_kind_for_codegen is Some then676    // alloc_error_handler_kind must also be Some.677    if tcx.alloc_error_handler_kind(()).unwrap() == AllocatorKind::Default {678        methods.push(AllocatorMethod {679            name: ALLOC_ERROR_HANDLER,680            special: None,681            inputs: &[AllocatorMethodInput { name: "layout", ty: AllocatorTy::Layout }],682            output: AllocatorTy::Never,683        });684    }685686    methods687}688689pub fn codegen_crate<B: ExtraBackendMethods>(690    backend: B,691    tcx: TyCtxt<'_>,692    crate_info: &CrateInfo,693) -> OngoingCodegen<B> {694    if tcx.sess.target.need_explicit_cpu && tcx.sess.opts.cg.target_cpu.is_none() {695        // The target has no default cpu, but none is set explicitly696        tcx.dcx().emit_fatal(errors::CpuRequired);697    }698699    if let Some(target_cpu) = &tcx.sess.opts.cg.target_cpu700        && tcx.sess.target.unsupported_cpus.contains(&target_cpu.into())701    {702        // The target cpu is explicitly listed as an unsupported cpu703        tcx.dcx().emit_fatal(errors::CpuUnsupported { target_cpu: target_cpu.clone() });704    }705706    let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);707708    // Run the monomorphization collector and partition the collected items into709    // codegen units.710    let MonoItemPartitions { codegen_units, .. } = tcx.collect_and_partition_mono_items(());711712    // Force all codegen_unit queries so they are already either red or green713    // when compile_codegen_unit accesses them. We are not able to re-execute714    // the codegen_unit query from just the DepNode, so an unknown color would715    // lead to having to re-execute compile_codegen_unit, possibly716    // unnecessarily.717    if tcx.dep_graph.is_fully_enabled() {718        for cgu in codegen_units {719            tcx.ensure_ok().codegen_unit(cgu.name());720        }721    }722723    // Codegen an allocator shim, if necessary.724    let allocator_module = if let Some(kind) = allocator_kind_for_codegen(tcx) {725        let llmod_id =726            cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string();727728        tcx.sess.time("write_allocator_module", || {729            let module =730                backend.codegen_allocator(tcx, &llmod_id, &allocator_shim_contents(tcx, kind));731            Some(ModuleCodegen::new_allocator(llmod_id, module))732        })733    } else {734        None735    };736737    let ongoing_codegen = start_async_codegen(backend.clone(), tcx, crate_info, allocator_module);738739    // For better throughput during parallel processing by LLVM, we used to sort740    // CGUs largest to smallest. This would lead to better thread utilization741    // by, for example, preventing a large CGU from being processed last and742    // having only one LLVM thread working while the rest remained idle.743    //744    // However, this strategy would lead to high memory usage, as it meant the745    // LLVM-IR for all of the largest CGUs would be resident in memory at once.746    //747    // Instead, we can compromise by ordering CGUs such that the largest and748    // smallest are first, second largest and smallest are next, etc. If there749    // are large size variations, this can reduce memory usage significantly.750    let codegen_units: Vec<_> = {751        let mut sorted_cgus = codegen_units.iter().collect::<Vec<_>>();752        sorted_cgus.sort_by_key(|cgu| cmp::Reverse(cgu.size_estimate()));753754        let (first_half, second_half) = sorted_cgus.split_at(sorted_cgus.len() / 2);755        first_half.iter().interleave(second_half.iter().rev()).copied().collect()756    };757758    // Calculate the CGU reuse759    let cgu_reuse = tcx.sess.time("find_cgu_reuse", || {760        codegen_units.iter().map(|cgu| determine_cgu_reuse(tcx, cgu)).collect::<Vec<_>>()761    });762763    crate::assert_module_sources::assert_module_sources(tcx, &|cgu_reuse_tracker| {764        for (i, cgu) in codegen_units.iter().enumerate() {765            let cgu_reuse = cgu_reuse[i];766            cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);767        }768    });769770    let mut total_codegen_time = Duration::new(0, 0);771    let start_rss = tcx.sess.opts.unstable_opts.time_passes.then(|| get_resident_set_size());772773    // The non-parallel compiler can only translate codegen units to LLVM IR774    // on a single thread, leading to a staircase effect where the N LLVM775    // threads have to wait on the single codegen threads to generate work776    // for them. The parallel compiler does not have this restriction, so777    // we can pre-load the LLVM queue in parallel before handing off778    // coordination to the OnGoingCodegen scheduler.779    //780    // This likely is a temporary measure. Once we don't have to support the781    // non-parallel compiler anymore, we can compile CGUs end-to-end in782    // parallel and get rid of the complicated scheduling logic.783    let mut pre_compiled_cgus = if tcx.sess.threads() > 1 {784        tcx.sess.time("compile_first_CGU_batch", || {785            // Try to find one CGU to compile per thread.786            let cgus: Vec<_> = cgu_reuse787                .iter()788                .enumerate()789                .filter(|&(_, reuse)| reuse == &CguReuse::No)790                .take(tcx.sess.threads())791                .collect();792793            // Compile the found CGUs in parallel.794            let start_time = Instant::now();795796            let pre_compiled_cgus = par_map(cgus, |(i, _)| {797                let module = backend.compile_codegen_unit(tcx, codegen_units[i].name());798                (i, IntoDynSyncSend(module))799            });800801            total_codegen_time += start_time.elapsed();802803            pre_compiled_cgus804        })805    } else {806        FxHashMap::default()807    };808809    for (i, cgu) in codegen_units.iter().enumerate() {810        ongoing_codegen.wait_for_signal_to_codegen_item();811        ongoing_codegen.check_for_errors(tcx.sess);812813        let cgu_reuse = cgu_reuse[i];814815        match cgu_reuse {816            CguReuse::No => {817                let (module, cost) = if let Some(cgu) = pre_compiled_cgus.remove(&i) {818                    cgu.0819                } else {820                    let start_time = Instant::now();821                    let module = backend.compile_codegen_unit(tcx, cgu.name());822                    total_codegen_time += start_time.elapsed();823                    module824                };825                // This will unwind if there are errors, which triggers our `AbortCodegenOnDrop`826                // guard. Unfortunately, just skipping the `submit_codegened_module_to_llvm` makes827                // compilation hang on post-monomorphization errors.828                tcx.dcx().abort_if_errors();829830                submit_codegened_module_to_llvm(&ongoing_codegen.coordinator, module, cost);831            }832            CguReuse::PreLto => {833                submit_pre_lto_module_to_llvm(834                    tcx,835                    &ongoing_codegen.coordinator,836                    CachedModuleCodegen {837                        name: cgu.name().to_string(),838                        source: cgu.previous_work_product(tcx),839                    },840                );841            }842            CguReuse::PostLto => {843                submit_post_lto_module_to_llvm(844                    &ongoing_codegen.coordinator,845                    CachedModuleCodegen {846                        name: cgu.name().to_string(),847                        source: cgu.previous_work_product(tcx),848                    },849                );850            }851        }852    }853854    ongoing_codegen.codegen_finished(tcx);855856    // Since the main thread is sometimes blocked during codegen, we keep track857    // -Ztime-passes output manually.858    if tcx.sess.opts.unstable_opts.time_passes {859        let end_rss = get_resident_set_size();860861        print_time_passes_entry(862            "codegen_to_LLVM_IR",863            total_codegen_time,864            start_rss.unwrap(),865            end_rss,866            tcx.sess.opts.unstable_opts.time_passes_format,867        );868    }869870    ongoing_codegen.check_for_errors(tcx.sess);871    ongoing_codegen872}873874/// Returns whether a call from the current crate to the [`Instance`] would produce a call875/// from `compiler_builtins` to a symbol the linker must resolve.876///877/// Such calls from `compiler_bultins` are effectively impossible for the linker to handle. Some878/// linkers will optimize such that dead calls to unresolved symbols are not an error, but this is879/// not guaranteed. So we used this function in codegen backends to ensure we do not generate any880/// unlinkable calls.881///882/// Note that calls to LLVM intrinsics are uniquely okay because they won't make it to the linker.883pub fn is_call_from_compiler_builtins_to_upstream_monomorphization<'tcx>(884    tcx: TyCtxt<'tcx>,885    instance: Instance<'tcx>,886) -> bool {887    fn is_llvm_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool {888        if let Some(name) = tcx.codegen_fn_attrs(def_id).symbol_name {889            name.as_str().starts_with("llvm.")890        } else {891            false892        }893    }894895    let def_id = instance.def_id();896    !def_id.is_local()897        && tcx.is_compiler_builtins(LOCAL_CRATE)898        && !is_llvm_intrinsic(tcx, def_id)899        && !tcx.should_codegen_locally(instance)900}901902impl CrateInfo {903    pub fn new(tcx: TyCtxt<'_>, target_cpu: String) -> CrateInfo {904        let crate_types = tcx.crate_types().to_vec();905        let exported_symbols = crate_types906            .iter()907            .map(|&c| (c, crate::back::linker::exported_symbols(tcx, c)))908            .collect();909        let linked_symbols =910            crate_types.iter().map(|&c| (c, crate::back::linker::linked_symbols(tcx, c))).collect();911        let local_crate_name = tcx.crate_name(LOCAL_CRATE);912        let windows_subsystem = find_attr!(tcx, crate, WindowsSubsystem(kind) => *kind);913914        // This list is used when generating the command line to pass through to915        // system linker. The linker expects undefined symbols on the left of the916        // command line to be defined in libraries on the right, not the other way917        // around. For more info, see some comments in the add_used_library function918        // below.919        //920        // In order to get this left-to-right dependency ordering, we use the reverse921        // postorder of all crates putting the leaves at the rightmost positions.922        let mut compiler_builtins = None;923        let mut used_crates: Vec<_> = tcx924            .postorder_cnums(())925            .iter()926            .rev()927            .copied()928            .filter(|&cnum| {929                let link = !tcx.crate_dep_kind(cnum).macros_only();930                if link && tcx.is_compiler_builtins(cnum) {931                    compiler_builtins = Some(cnum);932                    return false;933                }934                link935            })936            .collect();937        // `compiler_builtins` are always placed last to ensure that they're linked correctly.938        used_crates.extend(compiler_builtins);939940        let crates = tcx.crates(());941        let n_crates = crates.len();942        let mut info = CrateInfo {943            target_cpu,944            target_features: tcx.global_backend_features(()).clone(),945            crate_types,946            exported_symbols,947            linked_symbols,948            local_crate_name,949            compiler_builtins,950            profiler_runtime: None,951            is_no_builtins: Default::default(),952            native_libraries: Default::default(),953            used_libraries: tcx.native_libraries(LOCAL_CRATE).iter().map(Into::into).collect(),954            crate_name: UnordMap::with_capacity(n_crates),955            used_crates,956            used_crate_source: UnordMap::with_capacity(n_crates),957            dependency_formats: Arc::clone(tcx.dependency_formats(())),958            windows_subsystem,959            natvis_debugger_visualizers: Default::default(),960            lint_levels: CodegenLintLevels::from_tcx(tcx),961            metadata_symbol: exported_symbols::metadata_symbol_name(tcx),962        };963964        info.native_libraries.reserve(n_crates);965966        for &cnum in crates.iter() {967            info.native_libraries968                .insert(cnum, tcx.native_libraries(cnum).iter().map(Into::into).collect());969            info.crate_name.insert(cnum, tcx.crate_name(cnum));970971            let used_crate_source = tcx.used_crate_source(cnum);972            info.used_crate_source.insert(cnum, Arc::clone(used_crate_source));973            if tcx.is_profiler_runtime(cnum) {974                info.profiler_runtime = Some(cnum);975            }976            if tcx.is_no_builtins(cnum) {977                info.is_no_builtins.insert(cnum);978            }979        }980981        // Handle circular dependencies in the standard library.982        // See comment before `add_linked_symbol_object` function for the details.983        // If global LTO is enabled then almost everything (*) is glued into a single object file,984        // so this logic is not necessary and can cause issues on some targets (due to weak lang985        // item symbols being "privatized" to that object file), so we disable it.986        // (*) Native libs, and `#[compiler_builtins]` and `#[no_builtins]` crates are not glued,987        // and we assume that they cannot define weak lang items. This is not currently enforced988        // by the compiler, but that's ok because all this stuff is unstable anyway.989        let target = &tcx.sess.target;990        if !are_upstream_rust_objects_already_included(tcx.sess) {991            let add_prefix = match (target.is_like_windows, &target.arch) {992                (true, Arch::X86) => |name: String, _: SymbolExportKind| format!("_{name}"),993                (true, Arch::Arm64EC) => {994                    // Only functions are decorated for arm64ec.995                    |name: String, export_kind: SymbolExportKind| match export_kind {996                        SymbolExportKind::Text => format!("#{name}"),997                        _ => name,998                    }999                }1000                _ => |name: String, _: SymbolExportKind| name,1001            };1002            let missing_weak_lang_items: FxIndexSet<(Symbol, SymbolExportKind)> = info1003                .used_crates1004                .iter()1005                .flat_map(|&cnum| tcx.missing_lang_items(cnum))1006                .filter(|l| l.is_weak())1007                .filter_map(|&l| {1008                    let name = l.link_name()?;1009                    let export_kind = match l.target() {1010                        Target::Fn => SymbolExportKind::Text,1011                        Target::Static => SymbolExportKind::Data,1012                        _ => bug!(1013                            "Don't know what the export kind is for lang item of kind {:?}",1014                            l.target()1015                        ),1016                    };1017                    lang_items::required(tcx, l).then_some((name, export_kind))1018                })1019                .collect();10201021            // This loop only adds new items to values of the hash map, so the order in which we1022            // iterate over the values is not important.1023            #[allow(rustc::potential_query_instability)]1024            info.linked_symbols1025                .iter_mut()1026                .filter(|(crate_type, _)| {1027                    !matches!(crate_type, CrateType::Rlib | CrateType::StaticLib)1028                })1029                .for_each(|(_, linked_symbols)| {1030                    let mut symbols = missing_weak_lang_items1031                        .iter()1032                        .map(|(item, export_kind)| {1033                            (1034                                add_prefix(1035                                    mangle_internal_symbol(tcx, item.as_str()),1036                                    *export_kind,1037                                ),1038                                *export_kind,1039                            )1040                        })1041                        .collect::<Vec<_>>();1042                    symbols.sort_unstable_by(|a, b| a.0.cmp(&b.0));1043                    linked_symbols.extend(symbols);1044                });1045        }10461047        let embed_visualizers = tcx.crate_types().iter().any(|&crate_type| match crate_type {1048            CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Sdylib => {1049                // These are crate types for which we invoke the linker and can embed1050                // NatVis visualizers.1051                true1052            }1053            CrateType::ProcMacro => {1054                // We could embed NatVis for proc macro crates too (to improve the debugging1055                // experience for them) but it does not seem like a good default, since1056                // this is a rare use case and we don't want to slow down the common case.1057                false1058            }1059            CrateType::StaticLib | CrateType::Rlib => {1060                // We don't invoke the linker for these, so we don't need to collect the NatVis for1061                // them.1062                false1063            }1064        });10651066        if target.is_like_msvc && embed_visualizers {1067            info.natvis_debugger_visualizers =1068                collect_debugger_visualizers_transitive(tcx, DebuggerVisualizerType::Natvis);1069        }10701071        info1072    }1073}10741075pub(crate) fn provide(providers: &mut Providers) {1076    providers.backend_optimization_level = |tcx, cratenum| {1077        let for_speed = match tcx.sess.opts.optimize {1078            // If globally no optimisation is done, #[optimize] has no effect.1079            //1080            // This is done because if we ended up "upgrading" to `-O2` here, we’d populate the1081            // pass manager and it is likely that some module-wide passes (such as inliner or1082            // cross-function constant propagation) would ignore the `optnone` annotation we put1083            // on the functions, thus necessarily involving these functions into optimisations.1084            config::OptLevel::No => return config::OptLevel::No,1085            // If globally optimise-speed is already specified, just use that level.1086            config::OptLevel::Less => return config::OptLevel::Less,1087            config::OptLevel::More => return config::OptLevel::More,1088            config::OptLevel::Aggressive => return config::OptLevel::Aggressive,1089            // If globally optimize-for-size has been requested, use -O2 instead (if optimize(size)1090            // are present).1091            config::OptLevel::Size => config::OptLevel::More,1092            config::OptLevel::SizeMin => config::OptLevel::More,1093        };10941095        let defids = tcx.collect_and_partition_mono_items(cratenum).all_mono_items;10961097        let any_for_speed = defids.items().any(|id| {1098            let CodegenFnAttrs { optimize, .. } = tcx.codegen_fn_attrs(*id);1099            matches!(optimize, OptimizeAttr::Speed)1100        });11011102        if any_for_speed {1103            return for_speed;1104        }11051106        tcx.sess.opts.optimize1107    };1108}11091110pub fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguReuse {1111    if !tcx.dep_graph.is_fully_enabled() {1112        return CguReuse::No;1113    }11141115    let work_product_id = &cgu.work_product_id();1116    if tcx.dep_graph.previous_work_product(work_product_id).is_none() {1117        // We don't have anything cached for this CGU. This can happen1118        // if the CGU did not exist in the previous session.1119        return CguReuse::No;1120    }11211122    // Try to mark the CGU as green. If it we can do so, it means that nothing1123    // affecting the LLVM module has changed and we can re-use a cached version.1124    // If we compile with any kind of LTO, this means we can re-use the bitcode1125    // of the Pre-LTO stage (possibly also the Post-LTO version but we'll only1126    // know that later). If we are not doing LTO, there is only one optimized1127    // version of each module, so we re-use that.1128    let dep_node = cgu.codegen_dep_node(tcx);1129    tcx.dep_graph.assert_dep_node_not_yet_allocated_in_current_session(tcx.sess, &dep_node, || {1130        format!(1131            "CompileCodegenUnit dep-node for CGU `{}` already exists before marking.",1132            cgu.name()1133        )1134    });11351136    if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() {1137        // We can re-use either the pre- or the post-thinlto state. If no LTO is1138        // being performed then we can use post-LTO artifacts, otherwise we must1139        // reuse pre-LTO artifacts1140        match compute_per_cgu_lto_type(1141            &tcx.sess.lto(),1142            tcx.sess.opts.cg.linker_plugin_lto.enabled(),1143            tcx.crate_types(),1144        ) {1145            ComputedLtoType::No => CguReuse::PostLto,1146            _ => CguReuse::PreLto,1147        }1148    } else {1149        CguReuse::No1150    }1151}