1//! Handling of `static`s, `const`s and promoted allocations23use std::cmp::Ordering;45use cranelift_module::*;6use rustc_const_eval::interpret::CTFE_ALLOC_SALT;7use rustc_data_structures::fx::FxHashSet;8use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;9use rustc_middle::mir::interpret::{10 AllocId, GlobalAlloc, PointerArithmetic, Scalar, read_target_uint,11};12use rustc_middle::ty::{ExistentialTraitRef, ScalarInt};1314use crate::prelude::*;1516pub(crate) struct ConstantCx {17 todo: Vec<TodoItem>,18 anon_allocs: FxHashMap<AllocId, DataId>,19}2021#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]22enum TodoItem {23 Alloc(AllocId),24 Static(DefId),25}2627impl ConstantCx {28 pub(crate) fn new() -> Self {29 ConstantCx { todo: vec![], anon_allocs: FxHashMap::default() }30 }3132 pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) {33 define_all_allocs(tcx, module, &mut self);34 }35}3637pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) -> DataId {38 let mut constants_cx = ConstantCx::new();39 constants_cx.todo.push(TodoItem::Static(def_id));40 constants_cx.finalize(tcx, module);4142 data_id_for_static(43 tcx, module, def_id, false,44 // For a declaration the stated mutability doesn't matter.45 false,46 )47}4849pub(crate) fn codegen_tls_ref<'tcx>(50 fx: &mut FunctionCx<'_, '_, 'tcx>,51 def_id: DefId,52 layout: TyAndLayout<'tcx>,53) -> CValue<'tcx> {54 let tls_ptr = if !def_id.is_local() && fx.tcx.needs_thread_local_shim(def_id) {55 let instance = ty::Instance {56 def: ty::InstanceKind::ThreadLocalShim(def_id),57 args: ty::GenericArgs::empty(),58 };59 let func_ref = fx.get_function_ref(instance);60 let call = fx.bcx.ins().call(func_ref, &[]);61 fx.bcx.func.dfg.first_result(call)62 } else {63 let data_id = data_id_for_static(64 fx.tcx, fx.module, def_id, false,65 // For a declaration the stated mutability doesn't matter.66 false,67 );68 let local_data_id = fx.module.declare_data_in_func(data_id, fx.bcx.func);69 if fx.clif_comments.enabled() {70 fx.add_comment(local_data_id, format!("tls {:?}", def_id));71 }72 fx.bcx.ins().tls_value(fx.pointer_type, local_data_id)73 };74 CValue::by_val(tls_ptr, layout)75}7677pub(crate) fn eval_mir_constant<'tcx>(78 fx: &FunctionCx<'_, '_, 'tcx>,79 constant: &ConstOperand<'tcx>,80) -> (ConstValue, Ty<'tcx>) {81 let cv = fx.monomorphize(constant.const_);82 // This cannot fail because we checked all required_consts in advance.83 let val = cv84 .eval(fx.tcx, ty::TypingEnv::fully_monomorphized(), constant.span)85 .expect("erroneous constant missed by mono item collection");86 (val, cv.ty())87}8889pub(crate) fn codegen_constant_operand<'tcx>(90 fx: &mut FunctionCx<'_, '_, 'tcx>,91 constant: &ConstOperand<'tcx>,92) -> CValue<'tcx> {93 let (const_val, ty) = eval_mir_constant(fx, constant);94 codegen_const_value(fx, const_val, ty)95}9697pub(crate) fn codegen_const_value<'tcx>(98 fx: &mut FunctionCx<'_, '_, 'tcx>,99 const_val: ConstValue,100 ty: Ty<'tcx>,101) -> CValue<'tcx> {102 let layout = fx.layout_of(ty);103 assert!(layout.is_sized(), "unsized const value");104105 if layout.is_zst() {106 return CValue::zst(layout);107 }108109 match const_val {110 ConstValue::ZeroSized => unreachable!(), // we already handled ZST above111 ConstValue::Scalar(x) => match x {112 Scalar::Int(int) => {113 if fx.clif_type(layout.ty).is_some() {114 CValue::const_val(fx, layout, int)115 } else {116 let raw_val = int.size().truncate(int.to_bits(int.size()));117 let val = match int.size().bytes() {118 1 => fx.bcx.ins().iconst(types::I8, raw_val as i64),119 2 => fx.bcx.ins().iconst(types::I16, raw_val as i64),120 4 => fx.bcx.ins().iconst(types::I32, raw_val as i64),121 8 => fx.bcx.ins().iconst(types::I64, raw_val as i64),122 16 => {123 let lsb = fx.bcx.ins().iconst(types::I64, raw_val as u64 as i64);124 let msb =125 fx.bcx.ins().iconst(types::I64, (raw_val >> 64) as u64 as i64);126 fx.bcx.ins().iconcat(lsb, msb)127 }128 _ => unreachable!(),129 };130131 // FIXME avoid this extra copy to the stack and directly write to the final132 // destination133 let place = CPlace::new_stack_slot(fx, layout);134 place.to_ptr().store(fx, val, MemFlags::trusted());135 place.to_cvalue(fx)136 }137 }138 Scalar::Ptr(ptr, _size) => {139 let (prov, offset) = ptr.prov_and_relative_offset();140 let alloc_id = prov.alloc_id();141 let base_addr = match fx.tcx.global_alloc(alloc_id) {142 GlobalAlloc::Memory(alloc) => {143 if alloc.inner().len() == 0 {144 fx.bcx.ins().iconst(fx.pointer_type, alloc.inner().align.bytes() as i64)145 } else {146 let data_id = data_id_for_alloc_id(147 &mut fx.constants_cx,148 fx.module,149 alloc_id,150 alloc.inner().mutability,151 );152 let local_data_id =153 fx.module.declare_data_in_func(data_id, fx.bcx.func);154 if fx.clif_comments.enabled() {155 fx.add_comment(local_data_id, format!("{:?}", alloc_id));156 }157 fx.bcx.ins().symbol_value(fx.pointer_type, local_data_id)158 }159 }160 GlobalAlloc::Function { instance, .. } => {161 let func_id = crate::abi::import_function(fx.tcx, fx.module, instance);162 let local_func_id = fx.module.declare_func_in_func(func_id, fx.bcx.func);163 fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)164 }165 GlobalAlloc::VTable(ty, dyn_ty) => {166 let data_id = data_id_for_vtable(167 fx.tcx,168 &mut fx.constants_cx,169 fx.module,170 ty,171 dyn_ty.principal().map(|principal| {172 fx.tcx.instantiate_bound_regions_with_erased(principal)173 }),174 );175 let local_data_id = fx.module.declare_data_in_func(data_id, fx.bcx.func);176 fx.bcx.ins().symbol_value(fx.pointer_type, local_data_id)177 }178 GlobalAlloc::TypeId { .. } => {179 return CValue::const_val(180 fx,181 layout,182 ScalarInt::try_from_target_usize(offset.bytes(), fx.tcx).unwrap(),183 );184 }185 GlobalAlloc::Static(def_id) => {186 assert!(fx.tcx.is_static(def_id));187 let data_id = data_id_for_static(188 fx.tcx, fx.module, def_id, false,189 // For a declaration the stated mutability doesn't matter.190 false,191 );192 let local_data_id = fx.module.declare_data_in_func(data_id, fx.bcx.func);193 if fx.clif_comments.enabled() {194 fx.add_comment(local_data_id, format!("{:?}", def_id));195 }196 if fx197 .tcx198 .codegen_fn_attrs(def_id)199 .flags200 .contains(CodegenFnAttrFlags::THREAD_LOCAL)201 {202 fx.bcx.ins().tls_value(fx.pointer_type, local_data_id)203 } else {204 fx.bcx.ins().symbol_value(fx.pointer_type, local_data_id)205 }206 }207 };208 let val = if offset.bytes() != 0 {209 fx.bcx210 .ins()211 .iadd_imm(base_addr, fx.tcx.truncate_to_target_usize(offset.bytes()) as i64)212 } else {213 base_addr214 };215 CValue::by_val(val, layout)216 }217 },218 ConstValue::Indirect { alloc_id, offset } => CValue::by_ref(219 Pointer::new(pointer_for_allocation(fx, alloc_id))220 .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()),221 layout,222 ),223 ConstValue::Slice { alloc_id, meta } => {224 let ptr = pointer_for_allocation(fx, alloc_id);225 let len = fx.bcx.ins().iconst(fx.pointer_type, meta as i64);226 CValue::by_val_pair(ptr, len, layout)227 }228 }229}230231fn pointer_for_allocation<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, alloc_id: AllocId) -> Value {232 let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory();233 let data_id =234 data_id_for_alloc_id(&mut fx.constants_cx, fx.module, alloc_id, alloc.inner().mutability);235236 let local_data_id = fx.module.declare_data_in_func(data_id, fx.bcx.func);237 if fx.clif_comments.enabled() {238 fx.add_comment(local_data_id, format!("{:?}", alloc_id));239 }240 fx.bcx.ins().symbol_value(fx.pointer_type, local_data_id)241}242243fn data_id_for_alloc_id(244 cx: &mut ConstantCx,245 module: &mut dyn Module,246 alloc_id: AllocId,247 mutability: rustc_hir::Mutability,248) -> DataId {249 cx.todo.push(TodoItem::Alloc(alloc_id));250 *cx.anon_allocs251 .entry(alloc_id)252 .or_insert_with(|| module.declare_anonymous_data(mutability.is_mut(), false).unwrap())253}254255pub(crate) fn data_id_for_vtable<'tcx>(256 tcx: TyCtxt<'tcx>,257 cx: &mut ConstantCx,258 module: &mut dyn Module,259 ty: Ty<'tcx>,260 trait_ref: Option<ExistentialTraitRef<'tcx>>,261) -> DataId {262 let alloc_id = tcx.vtable_allocation((ty, trait_ref));263 data_id_for_alloc_id(cx, module, alloc_id, Mutability::Not)264}265266pub(crate) fn pointer_for_anonymous_str(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) -> Value {267 let alloc_id = fx.tcx.allocate_bytes_dedup(msg.as_bytes(), CTFE_ALLOC_SALT);268 pointer_for_allocation(fx, alloc_id)269}270271fn data_id_for_static(272 tcx: TyCtxt<'_>,273 module: &mut dyn Module,274 def_id: DefId,275 definition: bool,276 definition_writable: bool,277) -> DataId {278 let attrs = tcx.codegen_fn_attrs(def_id);279280 let instance = Instance::mono(tcx, def_id);281 let symbol_name = tcx.symbol_name(instance).name;282283 if let Some(import_linkage) = attrs.import_linkage {284 assert!(!definition);285 assert!(!tcx.is_mutable_static(def_id));286287 let ty = instance.ty(tcx, ty::TypingEnv::fully_monomorphized());288 let align = tcx289 .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty))290 .unwrap()291 .align292 .abi293 .bytes();294295 let linkage = if import_linkage == rustc_hir::attrs::Linkage::ExternalWeak296 || import_linkage == rustc_hir::attrs::Linkage::WeakAny297 {298 Linkage::Preemptible299 } else {300 Linkage::Import301 };302303 let data_id = match module.declare_data(304 symbol_name,305 linkage,306 false,307 attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL),308 ) {309 Ok(data_id) => data_id,310 Err(ModuleError::IncompatibleDeclaration(_)) => tcx.dcx().fatal(format!(311 "attempt to declare `{symbol_name}` as static, but it was already declared as function"312 )),313 Err(err) => Err::<_, _>(err).unwrap(),314 };315316 // Comment copied from https://github.com/rust-lang/rust/blob/45060c2a66dfd667f88bd8b94261b28a58d85bd5/src/librustc_codegen_llvm/consts.rs#L141317 // Declare an internal global `extern_with_linkage_foo` which318 // is initialized with the address of `foo`. If `foo` is319 // discarded during linking (for example, if `foo` has weak320 // linkage and there are no definitions), then321 // `extern_with_linkage_foo` will instead be initialized to322 // zero.323324 let ref_name = format!(325 "_rust_extern_with_linkage_{:016x}_{symbol_name}",326 tcx.stable_crate_id(LOCAL_CRATE)327 );328 let ref_data_id = module.declare_data(&ref_name, Linkage::Local, false, false).unwrap();329 let mut data = DataDescription::new();330 data.set_align(align);331 let data_gv = module.declare_data_in_data(data_id, &mut data);332 data.define(std::iter::repeat_n(0, pointer_ty(tcx).bytes() as usize).collect());333 data.write_data_addr(0, data_gv, 0);334 match module.define_data(ref_data_id, &data) {335 // Every time the static is referenced there will be another definition of this global,336 // so duplicate definitions are expected and allowed.337 Err(ModuleError::DuplicateDefinition(_)) => {}338 res => res.unwrap(),339 }340341 return ref_data_id;342 }343344 let linkage = if definition {345 crate::linkage::get_static_linkage(tcx, def_id)346 } else if attrs.linkage == Some(rustc_hir::attrs::Linkage::ExternalWeak)347 || attrs.linkage == Some(rustc_hir::attrs::Linkage::WeakAny)348 {349 Linkage::Preemptible350 } else {351 Linkage::Import352 };353354 match module.declare_data(355 symbol_name,356 linkage,357 definition_writable,358 attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL),359 ) {360 Ok(data_id) => data_id,361 Err(ModuleError::IncompatibleDeclaration(_)) => tcx.dcx().fatal(format!(362 "attempt to declare `{symbol_name}` as static, but it was already declared as function"363 )),364 Err(err) => Err::<_, _>(err).unwrap(),365 }366}367368fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut ConstantCx) {369 let mut done = FxHashSet::default();370 while let Some(todo_item) = cx.todo.pop() {371 if !done.insert(todo_item) {372 continue;373 }374375 let mut data = DataDescription::new();376377 let (data_id, alloc, section_name) = match todo_item {378 TodoItem::Alloc(alloc_id) => {379 let alloc = match tcx.global_alloc(alloc_id) {380 GlobalAlloc::Memory(alloc) => alloc,381 GlobalAlloc::Function { .. }382 | GlobalAlloc::Static(_)383 | GlobalAlloc::TypeId { .. }384 | GlobalAlloc::VTable(..) => {385 unreachable!()386 }387 };388 // FIXME: should we have a cache so we don't do this multiple times for the same `ConstAllocation`?389 let data_id = *cx.anon_allocs.entry(alloc_id).or_insert_with(|| {390 module.declare_anonymous_data(alloc.inner().mutability.is_mut(), false).unwrap()391 });392 (data_id, alloc, None)393 }394 TodoItem::Static(def_id) => {395 let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);396 let section_name = codegen_fn_attrs.link_section;397398 data.set_used(codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER));399400 let alloc = tcx.eval_static_initializer(def_id).unwrap();401402 let data_id = data_id_for_static(403 tcx,404 module,405 def_id,406 true,407 alloc.inner().mutability == Mutability::Mut,408 );409 (data_id, alloc, section_name)410 }411 };412413 let alloc = alloc.inner();414 data.set_align(alloc.align.bytes());415416 if let Some(section_name) = section_name {417 let (segment_name, section_name) = if tcx.sess.target.is_like_darwin {418 // See https://github.com/llvm/llvm-project/blob/main/llvm/lib/MC/MCSectionMachO.cpp419 let mut parts = section_name.as_str().split(',');420 let Some(segment_name) = parts.next() else {421 tcx.dcx().fatal(format!(422 "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma",423 section_name424 ));425 };426 let Some(section_name) = parts.next() else {427 tcx.dcx().fatal(format!(428 "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma",429 section_name430 ));431 };432 if section_name.len() > 16 {433 tcx.dcx().fatal(format!(434 "#[link_section = \"{}\"] is not valid for macos target: section name bigger than 16 bytes",435 section_name436 ));437 }438 let section_type = parts.next().unwrap_or("regular");439 if section_type != "regular" && section_type != "cstring_literals" {440 tcx.dcx().fatal(format!(441 "#[link_section = \"{}\"] is not supported: unsupported section type {}",442 section_name, section_type,443 ));444 }445 let _attrs = parts.next();446 if parts.next().is_some() {447 tcx.dcx().fatal(format!(448 "#[link_section = \"{}\"] is not valid for macos target: too many components",449 section_name450 ));451 }452 // FIXME(bytecodealliance/wasmtime#8901) set S_CSTRING_LITERALS section type when453 // cstring_literals is specified454 (segment_name, section_name)455 } else {456 ("", section_name.as_str())457 };458 data.set_segment_section(segment_name, section_name);459 }460461 let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();462 data.define(bytes.into_boxed_slice());463464 for &(offset, prov) in alloc.provenance().ptrs().iter() {465 let alloc_id = prov.alloc_id();466 let addend = {467 let endianness = tcx.data_layout.endian;468 let offset = offset.bytes() as usize;469 let ptr_size = tcx.data_layout.pointer_size();470 let bytes = &alloc.inspect_with_uninit_and_ptr_outside_interpreter(471 offset..offset + ptr_size.bytes() as usize,472 );473 read_target_uint(endianness, bytes).unwrap()474 };475476 let reloc_target_alloc = tcx.global_alloc(alloc_id);477 let data_id = match reloc_target_alloc {478 GlobalAlloc::Function { instance, .. } => {479 assert_eq!(addend, 0);480 let func_id = crate::abi::import_function(tcx, module, instance);481 let local_func_id = module.declare_func_in_data(func_id, &mut data);482 data.write_function_addr(offset.bytes() as u32, local_func_id);483 continue;484 }485 GlobalAlloc::Memory(target_alloc) => {486 data_id_for_alloc_id(cx, module, alloc_id, target_alloc.inner().mutability)487 }488 GlobalAlloc::VTable(ty, dyn_ty) => data_id_for_vtable(489 tcx,490 cx,491 module,492 ty,493 dyn_ty494 .principal()495 .map(|principal| tcx.instantiate_bound_regions_with_erased(principal)),496 ),497 GlobalAlloc::TypeId { .. } => {498 // Nothing to do, the bytes/offset of this pointer have already been written together with all other bytes,499 // so we just need to drop this provenance.500 continue;501 }502 GlobalAlloc::Static(def_id) => {503 if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)504 {505 tcx.dcx().fatal(format!(506 "Allocation {:?} contains reference to TLS value {:?}",507 alloc_id, def_id508 ));509 }510511 // Don't push a `TodoItem::Static` here, as it will cause statics used by512 // multiple crates to be duplicated between them. It isn't necessary anyway,513 // as it will get pushed by `codegen_static` when necessary.514 data_id_for_static(515 tcx, module, def_id, false,516 // For a declaration the stated mutability doesn't matter.517 false,518 )519 }520 };521522 let global_value = module.declare_data_in_data(data_id, &mut data);523 data.write_data_addr(offset.bytes() as u32, global_value, addend as i64);524 }525526 module.define_data(data_id, &data).unwrap();527 }528529 assert!(cx.todo.is_empty(), "{:?}", cx.todo);530}531532/// Used only for intrinsic implementations that need a compile-time constant533///534/// All uses of this function are a bug inside stdarch. [`eval_mir_constant`]535/// should be used everywhere, but for some vendor intrinsics stdarch forgets536/// to wrap the immediate argument in `const {}`, necesitating this hack to get537/// the correct value at compile time instead.538pub(crate) fn mir_operand_get_const_val<'tcx>(539 fx: &FunctionCx<'_, '_, 'tcx>,540 operand: &Operand<'tcx>,541) -> Option<ScalarInt> {542 match operand {543 Operand::RuntimeChecks(checks) => Some(checks.value(fx.tcx.sess).into()),544 Operand::Constant(const_) => eval_mir_constant(fx, const_).0.try_to_scalar_int(),545 // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored546 // inside a temporary before being passed to the intrinsic requiring the const argument.547 // This code tries to find a single constant defining definition of the referenced local.548 Operand::Copy(place) | Operand::Move(place) => {549 if !place.projection.is_empty() {550 return None;551 }552 let mut computed_scalar_int = None;553 for bb_data in fx.mir.basic_blocks.iter() {554 for stmt in &bb_data.statements {555 match &stmt.kind {556 StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => {557 match &local_and_rvalue.1 {558 Rvalue::Cast(559 CastKind::IntToInt560 | CastKind::FloatToFloat561 | CastKind::FloatToInt562 | CastKind::IntToFloat563 | CastKind::FnPtrToPtr564 | CastKind::PtrToPtr,565 operand,566 ty,567 ) => {568 if computed_scalar_int.is_some() {569 return None; // local assigned twice570 }571 if !matches!(ty.kind(), ty::Uint(_) | ty::Int(_)) {572 return None;573 }574 let scalar_int = mir_operand_get_const_val(fx, operand)?;575 let scalar_int =576 match fx.layout_of(*ty).size.cmp(&scalar_int.size()) {577 Ordering::Equal => scalar_int,578 Ordering::Less => match ty.kind() {579 ty::Uint(_) => ScalarInt::try_from_uint(580 scalar_int.to_uint(scalar_int.size()),581 fx.layout_of(*ty).size,582 )583 .unwrap(),584 ty::Int(_) => ScalarInt::try_from_int(585 scalar_int.to_int(scalar_int.size()),586 fx.layout_of(*ty).size,587 )588 .unwrap(),589 _ => unreachable!(),590 },591 Ordering::Greater => return None,592 };593 computed_scalar_int = Some(scalar_int);594 }595 Rvalue::Use(operand, _) => {596 computed_scalar_int = mir_operand_get_const_val(fx, operand)597 }598 _ => return None,599 }600 }601 StatementKind::SetDiscriminant { place: stmt_place, variant_index: _ }602 if &**stmt_place == place =>603 {604 return None;605 }606 StatementKind::Intrinsic(intrinsic) => match **intrinsic {607 NonDivergingIntrinsic::CopyNonOverlapping(..) => return None,608 NonDivergingIntrinsic::Assume(..) => {}609 },610 // conservative handling611 StatementKind::Assign(_)612 | StatementKind::FakeRead(_)613 | StatementKind::SetDiscriminant { .. }614 | StatementKind::StorageLive(_)615 | StatementKind::StorageDead(_)616 | StatementKind::AscribeUserType(_, _)617 | StatementKind::PlaceMention(..)618 | StatementKind::Coverage(_)619 | StatementKind::ConstEvalCounter620 | StatementKind::BackwardIncompatibleDropHint { .. }621 | StatementKind::Nop => {}622 }623 }624 match &bb_data.terminator().kind {625 TerminatorKind::Goto { .. }626 | TerminatorKind::SwitchInt { .. }627 | TerminatorKind::UnwindResume628 | TerminatorKind::UnwindTerminate(_)629 | TerminatorKind::Return630 | TerminatorKind::Unreachable631 | TerminatorKind::Drop { .. }632 | TerminatorKind::Assert { .. } => {}633 TerminatorKind::Yield { .. }634 | TerminatorKind::CoroutineDrop635 | TerminatorKind::FalseEdge { .. }636 | TerminatorKind::FalseUnwind { .. } => unreachable!(),637 TerminatorKind::InlineAsm { .. } => return None,638 TerminatorKind::Call { destination, target: Some(_), .. }639 if destination == place =>640 {641 return None;642 }643 TerminatorKind::TailCall { .. } => return None,644 TerminatorKind::Call { .. } => {}645 }646 }647 computed_scalar_int648 }649 }650}