1//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a2//! standalone executable.34use std::env;5use std::fs::File;6use std::io::BufWriter;7use std::path::PathBuf;8use std::sync::Arc;9use std::thread::JoinHandle;1011use cranelift_object::{ObjectBuilder, ObjectModule};12use rustc_codegen_ssa::assert_module_sources::CguReuse;13use rustc_codegen_ssa::back::write::produce_final_output_artifacts;14use rustc_codegen_ssa::base::determine_cgu_reuse;15use rustc_codegen_ssa::{CompiledModule, CompiledModules, ModuleKind};16use rustc_data_structures::profiling::SelfProfilerRef;17use rustc_data_structures::stable_hasher::{StableHash, StableHashCtxt, StableHasher};18use rustc_data_structures::sync::{IntoDynSyncSend, par_map};19use rustc_hir::attrs::Linkage as RLinkage;20use rustc_middle::dep_graph::{WorkProduct, WorkProductId};21use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;22use rustc_middle::mono::{CodegenUnit, MonoItem, MonoItemData, Visibility};23use rustc_session::Session;24use rustc_session::config::{OutputFilenames, OutputType};2526use crate::base::CodegenedFunction;27use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};28use crate::debuginfo::TypeDebugContext;29use crate::global_asm::{GlobalAsmConfig, GlobalAsmContext};30use crate::prelude::*;31use crate::unwind_module::UnwindModule;3233fn disable_incr_cache() -> bool {34    env::var("CG_CLIF_DISABLE_INCR_CACHE").as_deref() == Ok("1")35}3637struct ModuleCodegenResult {38    module_regular: CompiledModule,39    module_global_asm: Option<CompiledModule>,40    existing_work_product: Option<(WorkProductId, WorkProduct)>,41}4243enum OngoingModuleCodegen {44    Sync(Result<ModuleCodegenResult, String>),45    Async(JoinHandle<Result<ModuleCodegenResult, String>>),46}4748impl StableHash for OngoingModuleCodegen {49    fn stable_hash<Hcx: StableHashCtxt>(&self, _: &mut Hcx, _: &mut StableHasher) {50        // do nothing51    }52}5354pub(crate) struct OngoingCodegen {55    modules: Vec<OngoingModuleCodegen>,56    allocator_module: Option<CompiledModule>,57    concurrency_limiter: ConcurrencyLimiter,58}5960impl OngoingCodegen {61    pub(crate) fn join(62        self,63        sess: &Session,64        outputs: &OutputFilenames,65    ) -> (CompiledModules, FxIndexMap<WorkProductId, WorkProduct>) {66        let mut work_products = FxIndexMap::default();67        let mut modules = vec![];68        let disable_incr_cache = disable_incr_cache();6970        for module_codegen in self.modules {71            let module_codegen_result = match module_codegen {72                OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result,73                OngoingModuleCodegen::Async(join_handle) => match join_handle.join() {74                    Ok(module_codegen_result) => module_codegen_result,75                    Err(panic) => std::panic::resume_unwind(panic),76                },77            };7879            let module_codegen_result = match module_codegen_result {80                Ok(module_codegen_result) => module_codegen_result,81                Err(err) => sess.dcx().fatal(err),82            };83            let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } =84                module_codegen_result;8586            if let Some((work_product_id, work_product)) = existing_work_product {87                work_products.insert(work_product_id, work_product);88            } else {89                let work_product = if disable_incr_cache {90                    None91                } else if let Some(module_global_asm) = &module_global_asm {92                    rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(93                        sess,94                        &module_regular.name,95                        &[96                            ("o", module_regular.object.as_ref().unwrap()),97                            ("asm.o", module_global_asm.object.as_ref().unwrap()),98                        ],99                        &[],100                    )101                } else {102                    rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(103                        sess,104                        &module_regular.name,105                        &[("o", module_regular.object.as_ref().unwrap())],106                        &[],107                    )108                };109                if let Some((work_product_id, work_product)) = work_product {110                    work_products.insert(work_product_id, work_product);111                }112            }113114            modules.push(module_regular);115            if let Some(module_global_asm) = module_global_asm {116                modules.push(module_global_asm);117            }118        }119120        self.concurrency_limiter.finished();121122        sess.dcx().abort_if_errors();123124        let compiled_modules = CompiledModules { modules, allocator_module: self.allocator_module };125126        produce_final_output_artifacts(sess, &compiled_modules, outputs);127128        (compiled_modules, work_products)129    }130}131132fn make_module(sess: &Session, name: String) -> UnwindModule<ObjectModule> {133    let isa = crate::build_isa(sess, false);134135    let mut builder =136        ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();137138    // Disable function sections by default on MSVC as it causes significant slowdowns with link.exe.139    // Maybe link.exe has exponential behavior when there are many sections with the same name? Also140    // explicitly disable it on MinGW as rustc already disables it by default on MinGW and as such141    // isn't tested. If rustc enables it in the future on MinGW, we can re-enable it too once it has142    // been on MinGW.143    let default_function_sections = sess.target.function_sections && !sess.target.is_like_windows;144    builder.per_function_section(145        sess.opts.unstable_opts.function_sections.unwrap_or(default_function_sections),146    );147148    UnwindModule::new(ObjectModule::new(builder), true)149}150151fn emit_cgu(152    output_filenames: &OutputFilenames,153    invocation_temp: Option<&str>,154    prof: &SelfProfilerRef,155    name: String,156    module: UnwindModule<ObjectModule>,157    debug: Option<DebugContext>,158    global_asm_object_file: Option<PathBuf>,159    producer: &str,160) -> Result<ModuleCodegenResult, String> {161    let mut product = module.finish();162163    if let Some(mut debug) = debug {164        debug.emit(&mut product);165    }166167    let module_regular = emit_module(168        output_filenames,169        invocation_temp,170        prof,171        product.object,172        ModuleKind::Regular,173        name.clone(),174        producer,175    )?;176177    Ok(ModuleCodegenResult {178        module_regular,179        module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule {180            name: format!("{name}.asm"),181            kind: ModuleKind::Regular,182            object: Some(global_asm_object_file),183            dwarf_object: None,184            bytecode: None,185            assembly: None,186            llvm_ir: None,187            links_from_incr_cache: Vec::new(),188        }),189        existing_work_product: None,190    })191}192193fn emit_module(194    output_filenames: &OutputFilenames,195    invocation_temp: Option<&str>,196    prof: &SelfProfilerRef,197    mut object: cranelift_object::object::write::Object<'_>,198    kind: ModuleKind,199    name: String,200    producer_str: &str,201) -> Result<CompiledModule, String> {202    if object.format() == cranelift_object::object::BinaryFormat::Elf {203        let comment_section = object.add_section(204            Vec::new(),205            b".comment".to_vec(),206            cranelift_object::object::SectionKind::OtherString,207        );208        let mut producer = vec![0];209        producer.extend(producer_str.as_bytes());210        producer.push(0);211        object.set_section_data(comment_section, producer, 1);212    }213214    let tmp_file = output_filenames.temp_path_for_cgu(OutputType::Object, &name, invocation_temp);215    let file = match File::create(&tmp_file) {216        Ok(file) => file,217        Err(err) => return Err(format!("error creating object file: {}", err)),218    };219220    let mut file = BufWriter::new(file);221    if let Err(err) = object.write_stream(&mut file) {222        return Err(format!("error writing object file: {}", err));223    }224    let file = match file.into_inner() {225        Ok(file) => file,226        Err(err) => return Err(format!("error writing object file: {}", err)),227    };228229    if prof.enabled() {230        prof.artifact_size(231            "object_file",232            tmp_file.file_name().unwrap().to_string_lossy(),233            file.metadata().unwrap().len(),234        );235    }236237    Ok(CompiledModule {238        name,239        kind,240        object: Some(tmp_file),241        dwarf_object: None,242        bytecode: None,243        assembly: None,244        llvm_ir: None,245        links_from_incr_cache: Vec::new(),246    })247}248249fn reuse_workproduct_for_cgu(250    tcx: TyCtxt<'_>,251    cgu: &CodegenUnit<'_>,252) -> Result<ModuleCodegenResult, String> {253    let work_product = cgu.previous_work_product(tcx);254    let obj_out_regular = tcx.output_filenames(()).temp_path_for_cgu(255        OutputType::Object,256        cgu.name().as_str(),257        tcx.sess.invocation_temp.as_deref(),258    );259    let source_file_regular = rustc_incremental::in_incr_comp_dir_sess(260        tcx.sess,261        work_product.saved_files.get("o").expect("no saved object file in work product"),262    );263264    if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) {265        return Err(format!(266            "unable to copy {} to {}: {}",267            source_file_regular.display(),268            obj_out_regular.display(),269            err270        ));271    }272273    let obj_out_global_asm =274        crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm");275    let source_file_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") {276        let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(tcx.sess, asm_o);277        if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm)278        {279            return Err(format!(280                "unable to copy {} to {}: {}",281                source_file_global_asm.display(),282                obj_out_global_asm.display(),283                err284            ));285        }286        Some(source_file_global_asm)287    } else {288        None289    };290291    Ok(ModuleCodegenResult {292        module_regular: CompiledModule {293            name: cgu.name().to_string(),294            kind: ModuleKind::Regular,295            object: Some(obj_out_regular),296            dwarf_object: None,297            bytecode: None,298            assembly: None,299            llvm_ir: None,300            links_from_incr_cache: vec![source_file_regular],301        },302        module_global_asm: source_file_global_asm.map(|source_file| CompiledModule {303            name: cgu.name().to_string(),304            kind: ModuleKind::Regular,305            object: Some(obj_out_global_asm),306            dwarf_object: None,307            bytecode: None,308            assembly: None,309            llvm_ir: None,310            links_from_incr_cache: vec![source_file],311        }),312        existing_work_product: Some((cgu.work_product_id(), work_product)),313    })314}315316fn codegen_cgu_content(317    tcx: TyCtxt<'_>,318    module: &mut dyn Module,319    cgu_name: rustc_span::Symbol,320) -> (Option<DebugContext>, Vec<CodegenedFunction>, String) {321    let _timer = tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str());322323    let cgu = tcx.codegen_unit(cgu_name);324    let mono_items = cgu.items_in_deterministic_order(tcx);325326    let mut debug_context = DebugContext::new(tcx, module.isa(), false, cgu_name.as_str());327    let mut global_asm = String::new();328    let mut type_dbg = TypeDebugContext::default();329    super::predefine_mono_items(tcx, module, &mono_items);330    let mut codegened_functions = vec![];331    for (mono_item, item_data) in mono_items {332        match mono_item {333            MonoItem::Fn(instance) => {334                let flags = tcx.codegen_instance_attrs(instance.def).flags;335                if flags.contains(CodegenFnAttrFlags::NAKED) {336                    rustc_codegen_ssa::mir::naked_asm::codegen_naked_asm(337                        &mut GlobalAsmContext { tcx, global_asm: &mut global_asm },338                        instance,339                        MonoItemData {340                            linkage: RLinkage::External,341                            visibility: if item_data.linkage == RLinkage::Internal {342                                Visibility::Hidden343                            } else {344                                item_data.visibility345                            },346                            ..item_data347                        },348                    );349                    continue;350                }351                let codegened_function = crate::base::codegen_fn(352                    tcx,353                    cgu_name,354                    debug_context.as_mut(),355                    &mut type_dbg,356                    Function::new(),357                    module,358                    instance,359                );360                codegened_functions.push(codegened_function);361            }362            MonoItem::Static(def_id) => {363                let data_id = crate::constant::codegen_static(tcx, module, def_id);364                if let Some(debug_context) = debug_context.as_mut() {365                    debug_context.define_static(tcx, &mut type_dbg, def_id, data_id);366                }367            }368            MonoItem::GlobalAsm(item_id) => {369                rustc_codegen_ssa::base::codegen_global_asm(370                    &mut GlobalAsmContext { tcx, global_asm: &mut global_asm },371                    item_id,372                );373            }374        }375    }376    crate::main_shim::maybe_create_entry_wrapper(tcx, module, false, cgu.is_primary());377378    (debug_context, codegened_functions, global_asm)379}380381fn module_codegen(382    tcx: TyCtxt<'_>,383    global_asm_config: Arc<GlobalAsmConfig>,384    cgu_name: rustc_span::Symbol,385    token: ConcurrencyLimiterToken,386) -> OngoingModuleCodegen {387    let mut module = make_module(tcx.sess, cgu_name.as_str().to_string());388389    let (mut debug_context, codegened_functions, mut global_asm) =390        codegen_cgu_content(tcx, &mut module, cgu_name);391392    let cgu_name = cgu_name.as_str().to_owned();393394    let producer = crate::debuginfo::producer(tcx.sess);395396    let profiler = tcx.prof.clone();397    let invocation_temp = tcx.sess.invocation_temp.clone();398    let output_filenames = tcx.output_filenames(()).clone();399    let should_write_ir = crate::pretty_clif::should_write_ir(tcx.sess);400401    OngoingModuleCodegen::Async(std::thread::spawn(move || {402        profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| {403            cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler(404                profiler.clone(),405            )));406407            let mut cached_context = Context::new();408            for codegened_func in codegened_functions {409                crate::base::compile_fn(410                    &profiler,411                    &output_filenames,412                    should_write_ir,413                    &mut cached_context,414                    &mut module,415                    debug_context.as_mut(),416                    &mut global_asm,417                    codegened_func,418                );419            }420        });421422        let global_asm_object_file =423            profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| {424                crate::global_asm::compile_global_asm(425                    &global_asm_config,426                    &cgu_name,427                    global_asm,428                    invocation_temp.as_deref(),429                )430            })?;431432        let codegen_result =433            profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| {434                emit_cgu(435                    &global_asm_config.output_filenames,436                    invocation_temp.as_deref(),437                    &profiler,438                    cgu_name,439                    module,440                    debug_context,441                    global_asm_object_file,442                    &producer,443                )444            });445        std::mem::drop(token);446        codegen_result447    }))448}449450fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> {451    let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string());452    let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module);453454    if created_alloc_shim {455        let product = allocator_module.finish();456457        match emit_module(458            tcx.output_filenames(()),459            tcx.sess.invocation_temp.as_deref(),460            &tcx.sess.prof,461            product.object,462            ModuleKind::Allocator,463            "allocator_shim".to_owned(),464            &crate::debuginfo::producer(tcx.sess),465        ) {466            Ok(allocator_module) => Some(allocator_module),467            Err(err) => tcx.dcx().fatal(err),468        }469    } else {470        None471    }472}473474pub(crate) fn run_aot(tcx: TyCtxt<'_>) -> Box<OngoingCodegen> {475    let cgus = tcx.collect_and_partition_mono_items(()).codegen_units;476477    if tcx.dep_graph.is_fully_enabled() {478        for cgu in cgus {479            tcx.ensure_ok().codegen_unit(cgu.name());480        }481    }482483    // Calculate the CGU reuse484    let cgu_reuse = tcx.sess.time("find_cgu_reuse", || {485        cgus.iter().map(|cgu| determine_cgu_reuse(tcx, cgu)).collect::<Vec<_>>()486    });487488    rustc_codegen_ssa::assert_module_sources::assert_module_sources(tcx, &|cgu_reuse_tracker| {489        for (i, cgu) in cgus.iter().enumerate() {490            let cgu_reuse = cgu_reuse[i];491            cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);492        }493    });494495    let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx));496497    let disable_incr_cache = disable_incr_cache();498    let (todo_cgus, done_cgus) =499        cgus.iter().enumerate().partition::<Vec<_>, _>(|&(i, _)| match cgu_reuse[i] {500            _ if disable_incr_cache => true,501            CguReuse::No => true,502            CguReuse::PreLto | CguReuse::PostLto => false,503        });504505    let concurrency_limiter = IntoDynSyncSend(ConcurrencyLimiter::new(todo_cgus.len()));506507    let modules: Vec<_> =508        tcx.sess.time("codegen mono items", || {509            let modules: Vec<_> = par_map(todo_cgus, |(_, cgu)| {510                let dep_node = cgu.codegen_dep_node(tcx);511                let (module, _) = tcx.dep_graph.with_task(512                    dep_node,513                    tcx,514                    || {515                        module_codegen(516                            tcx,517                            global_asm_config.clone(),518                            cgu.name(),519                            concurrency_limiter.acquire(tcx.dcx()),520                        )521                    },522                    Some(rustc_middle::dep_graph::hash_result),523                );524                IntoDynSyncSend(module)525            });526            modules527                .into_iter()528                .map(|module| module.0)529                .chain(done_cgus.into_iter().map(|(_, cgu)| {530                    OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu))531                }))532                .collect()533        });534535    let allocator_module = emit_allocator_module(tcx);536537    Box::new(OngoingCodegen {538        modules,539        allocator_module,540        concurrency_limiter: concurrency_limiter.0,541    })542}