1// tidy-alphabetical-start2#![cfg_attr(feature = "nightly", allow(internal_features))]3#![cfg_attr(feature = "nightly", feature(rustc_attrs))]4#![cfg_attr(feature = "nightly", feature(step_trait))]5// tidy-alphabetical-end67/*! ABI handling for rustc89## What is an "ABI"?1011Literally, "application binary interface", which means it is everything about how code interacts,12at the machine level, with other code. This means it technically covers all of the following:13- object binary format for e.g. relocations or offset tables14- in-memory layout of types15- procedure calling conventions1617When we discuss "ABI" in the context of rustc, we are probably discussing calling conventions.18To describe those `rustc_abi` also covers type layout, as it must for values passed on the stack.19Despite `rustc_abi` being about calling conventions, it is good to remember these usages exist.20You will encounter all of them and more if you study target-specific codegen enough!21Even in general conversation, when someone says "the Rust ABI is unstable", it may allude to22either or both of23- `repr(Rust)` types have a mostly-unspecified layout24- `extern "Rust" fn(A) -> R` has an unspecified calling convention2526## Crate Goal2728ABI is a foundational concept, so the `rustc_abi` crate serves as an equally foundational crate.29It cannot carry all details relevant to an ABI: those permeate code generation and linkage.30Instead, `rustc_abi` is intended to provide the interface for reasoning about the binary interface.31It should contain traits and types that other crates then use in their implementation.32For example, a platform's `extern "C" fn` calling convention will be implemented in `rustc_target`33but `rustc_abi` contains the types for calculating layout and describing register-passing.34This makes it easier to describe things in the same way across targets, codegen backends, and35even other Rust compilers, such as rust-analyzer!3637*/3839use std::fmt;40#[cfg(feature = "nightly")]41use std::iter::Step;42use std::num::{NonZeroUsize, ParseIntError};43use std::ops::{Add, AddAssign, Deref, Mul, RangeFull, Sub};44use std::range::RangeInclusive;45use std::str::FromStr;4647use bitflags::bitflags;48#[cfg(feature = "nightly")]49use rustc_data_structures::stable_hash::StableOrd;50#[cfg(feature = "nightly")]51use rustc_error_messages::{DiagArgValue, IntoDiagArg};52#[cfg(feature = "nightly")]53use rustc_errors::{Diag, DiagCtxtHandle, Diagnostic, EmissionGuarantee, Level, msg};54use rustc_hashes::Hash64;55use rustc_index::{Idx, IndexSlice, IndexVec};56#[cfg(feature = "nightly")]57use rustc_macros::{Decodable_NoContext, Encodable_NoContext, StableHash};58#[cfg(feature = "nightly")]59use rustc_span::{Symbol, sym};6061mod callconv;62mod canon_abi;63mod extern_abi;64mod layout;65#[cfg(test)]66mod tests;6768pub use callconv::{Heterogeneous, HomogeneousAggregate, Reg, RegKind};69pub use canon_abi::{ArmCall, CanonAbi, InterruptKind, X86Call};70#[cfg(feature = "nightly")]71pub use extern_abi::CVariadicStatus;72pub use extern_abi::{ExternAbi, all_names};73pub use layout::{FIRST_VARIANT, FieldIdx, LayoutCalculator, LayoutCalculatorError, VariantIdx};74#[cfg(feature = "nightly")]75pub use layout::{Layout, TyAbiInterface, TyAndLayout};7677#[derive(Clone, Copy, PartialEq, Eq, Default)]78#[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]79pub struct ReprFlags(u8);8081bitflags! {82    impl ReprFlags: u8 {83        const IS_C               = 1 << 0;84        const IS_SIMD            = 1 << 1;85        const IS_TRANSPARENT     = 1 << 2;86        /// Internal only for now. If true, don't reorder fields.87        /// On its own it does not prevent ABI optimizations.88        const IS_LINEAR          = 1 << 3;89        /// If true, the type's crate has opted into layout randomization.90        /// Other flags can still inhibit reordering and thus randomization.91        /// The seed stored in `ReprOptions.field_shuffle_seed`.92        const RANDOMIZE_LAYOUT   = 1 << 4;93        /// If true, the type is always passed indirectly by non-Rustic ABIs.94        /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.95        const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5;96        const IS_SCALABLE        = 1 << 6;97         // Any of these flags being set prevent field reordering optimisation.98        const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()99                                 | ReprFlags::IS_SIMD.bits()100                                 | ReprFlags::IS_SCALABLE.bits()101                                 | ReprFlags::IS_LINEAR.bits();102        const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();103    }104}105106// This is the same as `rustc_data_structures::external_bitflags_debug` but without the107// `rustc_data_structures` to make it build on stable.108impl std::fmt::Debug for ReprFlags {109    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {110        bitflags::parser::to_writer(self, f)111    }112}113114#[derive(Copy, Clone, Debug, Eq, PartialEq)]115#[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]116pub enum IntegerType {117    /// Pointer-sized integer type, i.e. `isize` and `usize`. The field shows signedness, e.g.118    /// `Pointer(true)` means `isize`.119    Pointer(bool),120    /// Fixed-sized integer type, e.g. `i8`, `u32`, `i128`. The bool field shows signedness, e.g.121    /// `Fixed(I8, false)` means `u8`.122    Fixed(Integer, bool),123}124125impl IntegerType {126    pub fn is_signed(&self) -> bool {127        match self {128            IntegerType::Pointer(b) => *b,129            IntegerType::Fixed(_, b) => *b,130        }131    }132}133134#[derive(Copy, Clone, Debug, Eq, PartialEq)]135#[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]136pub enum ScalableElt {137    /// `N` in `rustc_scalable_vector(N)` - the element count of the scalable vector138    ElementCount(u16),139    /// `rustc_scalable_vector` w/out `N`, used for tuple types of scalable vectors that only140    /// contain other scalable vectors141    Container,142}143144/// Represents the repr options provided by the user.145#[derive(Copy, Clone, Debug, Eq, PartialEq, Default)]146#[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]147pub struct ReprOptions {148    pub int: Option<IntegerType>,149    pub align: Option<Align>,150    pub pack: Option<Align>,151    pub flags: ReprFlags,152    /// `#[rustc_scalable_vector]`153    pub scalable: Option<ScalableElt>,154    /// The seed to be used for randomizing a type's layout155    ///156    /// Note: This could technically be a `u128` which would157    /// be the "most accurate" hash as it'd encompass the item and crate158    /// hash without loss, but it does pay the price of being larger.159    /// Everything's a tradeoff, a 64-bit seed should be sufficient for our160    /// purposes (primarily `-Z randomize-layout`)161    pub field_shuffle_seed: Hash64,162}163164impl ReprOptions {165    #[inline]166    pub fn simd(&self) -> bool {167        self.flags.contains(ReprFlags::IS_SIMD)168    }169170    #[inline]171    pub fn scalable(&self) -> bool {172        self.flags.contains(ReprFlags::IS_SCALABLE)173    }174175    #[inline]176    pub fn c(&self) -> bool {177        self.flags.contains(ReprFlags::IS_C)178    }179180    #[inline]181    pub fn packed(&self) -> bool {182        self.pack.is_some()183    }184185    #[inline]186    pub fn transparent(&self) -> bool {187        self.flags.contains(ReprFlags::IS_TRANSPARENT)188    }189190    #[inline]191    pub fn linear(&self) -> bool {192        self.flags.contains(ReprFlags::IS_LINEAR)193    }194195    /// Returns the discriminant type, given these `repr` options.196    /// This must only be called on enums!197    ///198    /// This is the "typeck type" of the discriminant, which is effectively the maximum size:199    /// discriminant values will be wrapped to fit (with a lint). Layout can later decide to use a200    /// smaller type for the tag that stores the discriminant at runtime and that will work just201    /// fine, it just induces casts when getting/setting the discriminant.202    pub fn discr_type(&self) -> IntegerType {203        self.int.unwrap_or(IntegerType::Pointer(true))204    }205206    /// Returns `true` if this `#[repr()]` should inhabit "smart enum207    /// layout" optimizations, such as representing `Foo<&T>` as a208    /// single pointer.209    pub fn inhibit_enum_layout_opt(&self) -> bool {210        self.c() || self.int.is_some()211    }212213    pub fn inhibit_newtype_abi_optimization(&self) -> bool {214        self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)215    }216217    /// Returns `true` if this `#[repr()]` guarantees a fixed field order,218    /// e.g. `repr(C)` or `repr(<int>)`.219    pub fn inhibit_struct_field_reordering(&self) -> bool {220        self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()221    }222223    /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`224    /// was enabled for its declaration crate.225    pub fn can_randomize_type_layout(&self) -> bool {226        !self.inhibit_struct_field_reordering() && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT)227    }228229    /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations.230    pub fn inhibits_union_abi_opt(&self) -> bool {231        self.c()232    }233}234235/// The maximum supported number of lanes in a SIMD vector.236///237/// This value is selected based on backend support:238/// * LLVM does not appear to have a vector width limit.239/// * Cranelift stores the base-2 log of the lane count in a 4 bit integer.240pub const MAX_SIMD_LANES: u64 = 1 << 0xF;241242/// How pointers are represented in a given address space243#[derive(Copy, Clone, Debug, PartialEq, Eq)]244pub struct PointerSpec {245    /// The size of the bitwise representation of the pointer.246    pointer_size: Size,247    /// The alignment of pointers for this address space248    pointer_align: Align,249    /// The size of the value a pointer can be offset by in this address space.250    pointer_offset: Size,251    /// Pointers into this address space contain extra metadata252    /// FIXME(workingjubilee): Consider adequately reflecting this in the compiler?253    _is_fat: bool,254}255256/// Parsed [Data layout](https://llvm.org/docs/LangRef.html#data-layout)257/// for a target, which contains everything needed to compute layouts.258#[derive(Debug, PartialEq, Eq)]259pub struct TargetDataLayout {260    pub endian: Endian,261    pub i1_align: Align,262    pub i8_align: Align,263    pub i16_align: Align,264    pub i32_align: Align,265    pub i64_align: Align,266    pub i128_align: Align,267    pub f16_align: Align,268    pub f32_align: Align,269    pub f64_align: Align,270    pub f128_align: Align,271    pub aggregate_align: Align,272273    /// Alignments for vector types.274    pub vector_align: Vec<(Size, Align)>,275276    pub default_address_space: AddressSpace,277    pub default_address_space_pointer_spec: PointerSpec,278279    /// Address space information of all known address spaces.280    ///281    /// # Note282    ///283    /// This vector does not contain the [`PointerSpec`] relative to the default address space,284    /// which instead lives in [`Self::default_address_space_pointer_spec`].285    address_space_info: Vec<(AddressSpace, PointerSpec)>,286287    pub instruction_address_space: AddressSpace,288289    /// Minimum size of #[repr(C)] enums (default c_int::BITS, usually 32)290    /// Note: This isn't in LLVM's data layout string, it is `short_enum`291    /// so the only valid spec for LLVM is c_int::BITS or 8292    pub c_enum_min_size: Integer,293}294295impl Default for TargetDataLayout {296    /// Creates an instance of `TargetDataLayout`.297    fn default() -> TargetDataLayout {298        let align = |bits| Align::from_bits(bits).unwrap();299        TargetDataLayout {300            endian: Endian::Big,301            i1_align: align(8),302            i8_align: align(8),303            i16_align: align(16),304            i32_align: align(32),305            i64_align: align(32),306            i128_align: align(32),307            f16_align: align(16),308            f32_align: align(32),309            f64_align: align(64),310            f128_align: align(128),311            aggregate_align: align(8),312            vector_align: vec![313                (Size::from_bits(64), align(64)),314                (Size::from_bits(128), align(128)),315            ],316            default_address_space: AddressSpace::ZERO,317            default_address_space_pointer_spec: PointerSpec {318                pointer_size: Size::from_bits(64),319                pointer_align: align(64),320                pointer_offset: Size::from_bits(64),321                _is_fat: false,322            },323            address_space_info: vec![],324            instruction_address_space: AddressSpace::ZERO,325            c_enum_min_size: Integer::I32,326        }327    }328}329330pub enum TargetDataLayoutError<'a> {331    InvalidAddressSpace { addr_space: &'a str, cause: &'a str, err: ParseIntError },332    InvalidBits { kind: &'a str, bit: &'a str, cause: &'a str, err: ParseIntError },333    MissingAlignment { cause: &'a str },334    InvalidAlignment { cause: &'a str, err: AlignFromBytesError },335    InconsistentTargetArchitecture { dl: &'a str, target: &'a str },336    InconsistentTargetPointerWidth { pointer_size: u64, target: u16 },337    InvalidBitsSize { err: String },338    UnknownPointerSpecification { err: String },339}340341#[cfg(feature = "nightly")]342impl<G: EmissionGuarantee> Diagnostic<'_, G> for TargetDataLayoutError<'_> {343    fn into_diag(self, dcx: DiagCtxtHandle<'_>, level: Level) -> Diag<'_, G> {344        match self {345            TargetDataLayoutError::InvalidAddressSpace { addr_space, err, cause } => {346                Diag::new(dcx, level, msg!("invalid address space `{$addr_space}` for `{$cause}` in \"data-layout\": {$err}"))347                    .with_arg("addr_space", addr_space)348                    .with_arg("cause", cause)349                    .with_arg("err", err)350            }351            TargetDataLayoutError::InvalidBits { kind, bit, cause, err } => {352                Diag::new(dcx, level, msg!("invalid {$kind} `{$bit}` for `{$cause}` in \"data-layout\": {$err}"))353                    .with_arg("kind", kind)354                    .with_arg("bit", bit)355                    .with_arg("cause", cause)356                    .with_arg("err", err)357            }358            TargetDataLayoutError::MissingAlignment { cause } => {359                Diag::new(dcx, level, msg!("missing alignment for `{$cause}` in \"data-layout\""))360                    .with_arg("cause", cause)361            }362            TargetDataLayoutError::InvalidAlignment { cause, err } => {363                Diag::new(dcx, level, msg!("invalid alignment for `{$cause}` in \"data-layout\": {$err}"))364                    .with_arg("cause", cause)365                    .with_arg("err", err.to_string())366            }367            TargetDataLayoutError::InconsistentTargetArchitecture { dl, target } => {368                Diag::new(dcx, level, msg!("inconsistent target specification: \"data-layout\" claims architecture is {$dl}-endian, while \"target-endian\" is `{$target}`"))369                    .with_arg("dl", dl).with_arg("target", target)370            }371            TargetDataLayoutError::InconsistentTargetPointerWidth { pointer_size, target } => {372                Diag::new(dcx, level, msg!("inconsistent target specification: \"data-layout\" claims pointers are {$pointer_size}-bit, while \"target-pointer-width\" is `{$target}`"))373                    .with_arg("pointer_size", pointer_size).with_arg("target", target)374            }375            TargetDataLayoutError::InvalidBitsSize { err } => {376                Diag::new(dcx, level, msg!("{$err}")).with_arg("err", err)377            }378            TargetDataLayoutError::UnknownPointerSpecification { err } => {379                Diag::new(dcx, level, msg!("unknown pointer specification `{$err}` in datalayout string"))380                    .with_arg("err", err)381            }382        }383    }384}385386impl TargetDataLayout {387    /// Parse data layout from an388    /// [llvm data layout string](https://llvm.org/docs/LangRef.html#data-layout)389    ///390    /// This function doesn't fill `c_enum_min_size` and it will always be `I32` since it can not be391    /// determined from llvm string.392    pub fn parse_from_llvm_datalayout_string<'a>(393        input: &'a str,394        default_address_space: AddressSpace,395    ) -> Result<TargetDataLayout, TargetDataLayoutError<'a>> {396        // Parse an address space index from a string.397        let parse_address_space = |s: &'a str, cause: &'a str| {398            s.parse::<u32>().map(AddressSpace).map_err(|err| {399                TargetDataLayoutError::InvalidAddressSpace { addr_space: s, cause, err }400            })401        };402403        // Parse a bit count from a string.404        let parse_bits = |s: &'a str, kind: &'a str, cause: &'a str| {405            s.parse::<u64>().map_err(|err| TargetDataLayoutError::InvalidBits {406                kind,407                bit: s,408                cause,409                err,410            })411        };412413        // Parse a size string.414        let parse_size =415            |s: &'a str, cause: &'a str| parse_bits(s, "size", cause).map(Size::from_bits);416417        // Parse an alignment string.418        let parse_align_str = |s: &'a str, cause: &'a str| {419            let align_from_bits = |bits| {420                Align::from_bits(bits)421                    .map_err(|err| TargetDataLayoutError::InvalidAlignment { cause, err })422            };423            let abi = parse_bits(s, "alignment", cause)?;424            Ok(align_from_bits(abi)?)425        };426427        // Parse an alignment sequence, possibly in the form `<align>[:<preferred_alignment>]`,428        // ignoring the secondary alignment specifications.429        let parse_align_seq = |s: &[&'a str], cause: &'a str| {430            if s.is_empty() {431                return Err(TargetDataLayoutError::MissingAlignment { cause });432            }433            parse_align_str(s[0], cause)434        };435436        let mut dl = TargetDataLayout::default();437        dl.default_address_space = default_address_space;438439        let mut i128_align_src = 64;440        for spec in input.split('-') {441            let spec_parts = spec.split(':').collect::<Vec<_>>();442443            match &*spec_parts {444                ["e"] => dl.endian = Endian::Little,445                ["E"] => dl.endian = Endian::Big,446                [p] if p.starts_with('P') => {447                    dl.instruction_address_space = parse_address_space(&p[1..], "P")?448                }449                ["a", a @ ..] => dl.aggregate_align = parse_align_seq(a, "a")?,450                ["f16", a @ ..] => dl.f16_align = parse_align_seq(a, "f16")?,451                ["f32", a @ ..] => dl.f32_align = parse_align_seq(a, "f32")?,452                ["f64", a @ ..] => dl.f64_align = parse_align_seq(a, "f64")?,453                ["f128", a @ ..] => dl.f128_align = parse_align_seq(a, "f128")?,454                [p, s, a @ ..] if p.starts_with("p") => {455                    let mut p = p.strip_prefix('p').unwrap();456                    let mut _is_fat = false;457458                    // Some targets, such as CHERI, use the 'f' suffix in the p- spec to signal that459                    // they use 'fat' pointers. The resulting prefix may look like `pf<addr_space>`.460461                    if p.starts_with('f') {462                        p = p.strip_prefix('f').unwrap();463                        _is_fat = true;464                    }465466                    // However, we currently don't take into account further specifications:467                    // an error is emitted instead.468                    if p.starts_with(char::is_alphabetic) {469                        return Err(TargetDataLayoutError::UnknownPointerSpecification {470                            err: p.to_string(),471                        });472                    }473474                    let addr_space = if !p.is_empty() {475                        parse_address_space(p, "p-")?476                    } else {477                        AddressSpace::ZERO478                    };479480                    let pointer_size = parse_size(s, "p-")?;481                    let pointer_align = parse_align_seq(a, "p-")?;482                    let info = PointerSpec {483                        pointer_offset: pointer_size,484                        pointer_size,485                        pointer_align,486                        _is_fat,487                    };488                    if addr_space == default_address_space {489                        dl.default_address_space_pointer_spec = info;490                    } else {491                        match dl.address_space_info.iter_mut().find(|(a, _)| *a == addr_space) {492                            Some(e) => e.1 = info,493                            None => {494                                dl.address_space_info.push((addr_space, info));495                            }496                        }497                    }498                }499                [p, s, a, _pr, i] if p.starts_with("p") => {500                    let mut p = p.strip_prefix('p').unwrap();501                    let mut _is_fat = false;502503                    // Some targets, such as CHERI, use the 'f' suffix in the p- spec to signal that504                    // they use 'fat' pointers. The resulting prefix may look like `pf<addr_space>`.505506                    if p.starts_with('f') {507                        p = p.strip_prefix('f').unwrap();508                        _is_fat = true;509                    }510511                    // However, we currently don't take into account further specifications:512                    // an error is emitted instead.513                    if p.starts_with(char::is_alphabetic) {514                        return Err(TargetDataLayoutError::UnknownPointerSpecification {515                            err: p.to_string(),516                        });517                    }518519                    let addr_space = if !p.is_empty() {520                        parse_address_space(p, "p")?521                    } else {522                        AddressSpace::ZERO523                    };524525                    let info = PointerSpec {526                        pointer_size: parse_size(s, "p-")?,527                        pointer_align: parse_align_str(a, "p-")?,528                        pointer_offset: parse_size(i, "p-")?,529                        _is_fat,530                    };531532                    if addr_space == default_address_space {533                        dl.default_address_space_pointer_spec = info;534                    } else {535                        match dl.address_space_info.iter_mut().find(|(a, _)| *a == addr_space) {536                            Some(e) => e.1 = info,537                            None => {538                                dl.address_space_info.push((addr_space, info));539                            }540                        }541                    }542                }543544                [s, a @ ..] if s.starts_with('i') => {545                    let Ok(bits) = s[1..].parse::<u64>() else {546                        parse_size(&s[1..], "i")?; // For the user error.547                        continue;548                    };549                    let a = parse_align_seq(a, s)?;550                    match bits {551                        1 => dl.i1_align = a,552                        8 => dl.i8_align = a,553                        16 => dl.i16_align = a,554                        32 => dl.i32_align = a,555                        64 => dl.i64_align = a,556                        _ => {}557                    }558                    if bits >= i128_align_src && bits <= 128 {559                        // Default alignment for i128 is decided by taking the alignment of560                        // largest-sized i{64..=128}.561                        i128_align_src = bits;562                        dl.i128_align = a;563                    }564                }565                [s, a @ ..] if s.starts_with('v') => {566                    let v_size = parse_size(&s[1..], "v")?;567                    let a = parse_align_seq(a, s)?;568                    if let Some(v) = dl.vector_align.iter_mut().find(|v| v.0 == v_size) {569                        v.1 = a;570                        continue;571                    }572                    // No existing entry, add a new one.573                    dl.vector_align.push((v_size, a));574                }575                _ => {} // Ignore everything else.576            }577        }578579        // Inherit, if not given, address space information for specific LLVM elements from the580        // default data address space.581        if (dl.instruction_address_space != dl.default_address_space)582            && dl583                .address_space_info584                .iter()585                .find(|(a, _)| *a == dl.instruction_address_space)586                .is_none()587        {588            dl.address_space_info.push((589                dl.instruction_address_space,590                dl.default_address_space_pointer_spec.clone(),591            ));592        }593594        Ok(dl)595    }596597    /// Returns **exclusive** upper bound on object size in bytes, in the default data address598    /// space.599    ///600    /// The theoretical maximum object size is defined as the maximum positive `isize` value.601    /// This ensures that the `offset` semantics remain well-defined by allowing it to correctly602    /// index every address within an object along with one byte past the end, along with allowing603    /// `isize` to store the difference between any two pointers into an object.604    ///605    /// LLVM uses a 64-bit integer to represent object size in *bits*, but we care only for bytes,606    /// so we adopt such a more-constrained size bound due to its technical limitations.607    #[inline]608    pub fn obj_size_bound(&self) -> u64 {609        match self.pointer_size().bits() {610            16 => 1 << 15,611            32 => 1 << 31,612            64 => 1 << 61,613            bits => panic!("obj_size_bound: unknown pointer bit size {bits}"),614        }615    }616617    /// Returns **exclusive** upper bound on object size in bytes.618    ///619    /// The theoretical maximum object size is defined as the maximum positive `isize` value.620    /// This ensures that the `offset` semantics remain well-defined by allowing it to correctly621    /// index every address within an object along with one byte past the end, along with allowing622    /// `isize` to store the difference between any two pointers into an object.623    ///624    /// LLVM uses a 64-bit integer to represent object size in *bits*, but we care only for bytes,625    /// so we adopt such a more-constrained size bound due to its technical limitations.626    #[inline]627    pub fn obj_size_bound_in(&self, address_space: AddressSpace) -> u64 {628        match self.pointer_size_in(address_space).bits() {629            16 => 1 << 15,630            32 => 1 << 31,631            64 => 1 << 61,632            bits => panic!("obj_size_bound: unknown pointer bit size {bits}"),633        }634    }635636    #[inline]637    pub fn ptr_sized_integer(&self) -> Integer {638        use Integer::*;639        match self.pointer_offset().bits() {640            16 => I16,641            32 => I32,642            64 => I64,643            bits => panic!("ptr_sized_integer: unknown pointer bit size {bits}"),644        }645    }646647    #[inline]648    pub fn ptr_sized_integer_in(&self, address_space: AddressSpace) -> Integer {649        use Integer::*;650        match self.pointer_offset_in(address_space).bits() {651            16 => I16,652            32 => I32,653            64 => I64,654            bits => panic!("ptr_sized_integer: unknown pointer bit size {bits}"),655        }656    }657658    /// psABI-mandated alignment for a vector type, if any659    #[inline]660    fn cabi_vector_align(&self, vec_size: Size) -> Option<Align> {661        self.vector_align662            .iter()663            .find(|(size, _align)| *size == vec_size)664            .map(|(_size, align)| *align)665    }666667    /// an alignment resembling the one LLVM would pick for a vector668    #[inline]669    pub fn llvmlike_vector_align(&self, vec_size: Size) -> Align {670        self.cabi_vector_align(vec_size)671            .unwrap_or(Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap())672    }673674    /// Get the pointer size in the default data address space.675    #[inline]676    pub fn pointer_size(&self) -> Size {677        self.default_address_space_pointer_spec.pointer_size678    }679680    /// Get the pointer size in a specific address space.681    #[inline]682    pub fn pointer_size_in(&self, c: AddressSpace) -> Size {683        if c == self.default_address_space {684            return self.default_address_space_pointer_spec.pointer_size;685        }686687        if let Some(e) = self.address_space_info.iter().find(|(a, _)| a == &c) {688            e.1.pointer_size689        } else {690            panic!("Use of unknown address space {c:?}");691        }692    }693694    /// Get the pointer index in the default data address space.695    #[inline]696    pub fn pointer_offset(&self) -> Size {697        self.default_address_space_pointer_spec.pointer_offset698    }699700    /// Get the pointer index in a specific address space.701    #[inline]702    pub fn pointer_offset_in(&self, c: AddressSpace) -> Size {703        if c == self.default_address_space {704            return self.default_address_space_pointer_spec.pointer_offset;705        }706707        if let Some(e) = self.address_space_info.iter().find(|(a, _)| a == &c) {708            e.1.pointer_offset709        } else {710            panic!("Use of unknown address space {c:?}");711        }712    }713714    /// Get the pointer alignment in the default data address space.715    #[inline]716    pub fn pointer_align(&self) -> AbiAlign {717        AbiAlign::new(self.default_address_space_pointer_spec.pointer_align)718    }719720    /// Get the pointer alignment in a specific address space.721    #[inline]722    pub fn pointer_align_in(&self, c: AddressSpace) -> AbiAlign {723        AbiAlign::new(if c == self.default_address_space {724            self.default_address_space_pointer_spec.pointer_align725        } else if let Some(e) = self.address_space_info.iter().find(|(a, _)| a == &c) {726            e.1.pointer_align727        } else {728            panic!("Use of unknown address space {c:?}");729        })730    }731}732733pub trait HasDataLayout {734    fn data_layout(&self) -> &TargetDataLayout;735}736737impl HasDataLayout for TargetDataLayout {738    #[inline]739    fn data_layout(&self) -> &TargetDataLayout {740        self741    }742}743744// used by rust-analyzer745impl HasDataLayout for &TargetDataLayout {746    #[inline]747    fn data_layout(&self) -> &TargetDataLayout {748        (**self).data_layout()749    }750}751752/// Endianness of the target, which must match cfg(target-endian).753#[derive(Copy, Clone, PartialEq, Eq)]754pub enum Endian {755    Little,756    Big,757}758759impl Endian {760    pub fn as_str(&self) -> &'static str {761        match self {762            Self::Little => "little",763            Self::Big => "big",764        }765    }766767    #[cfg(feature = "nightly")]768    pub fn desc_symbol(&self) -> Symbol {769        match self {770            Self::Little => sym::little,771            Self::Big => sym::big,772        }773    }774}775776impl fmt::Debug for Endian {777    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {778        f.write_str(self.as_str())779    }780}781782impl FromStr for Endian {783    type Err = String;784785    fn from_str(s: &str) -> Result<Self, Self::Err> {786        match s {787            "little" => Ok(Self::Little),788            "big" => Ok(Self::Big),789            _ => Err(format!(r#"unknown endian: "{s}""#)),790        }791    }792}793794/// Size of a type in bytes.795#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]796#[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]797pub struct Size {798    raw: u64,799}800801#[cfg(feature = "nightly")]802impl StableOrd for Size {803    const CAN_USE_UNSTABLE_SORT: bool = true;804805    // `Ord` is implemented as just comparing numerical values and numerical values806    // are not changed by (de-)serialization.807    const THIS_IMPLEMENTATION_HAS_BEEN_TRIPLE_CHECKED: () = ();808}809810// This is debug-printed a lot in larger structs, don't waste too much space there811impl fmt::Debug for Size {812    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {813        write!(f, "Size({} bytes)", self.bytes())814    }815}816817impl Size {818    pub const ZERO: Size = Size { raw: 0 };819820    /// Rounds `bits` up to the next-higher byte boundary, if `bits` is821    /// not a multiple of 8.822    pub fn from_bits(bits: impl TryInto<u64>) -> Size {823        let bits = bits.try_into().ok().unwrap();824        Size { raw: bits.div_ceil(8) }825    }826827    #[inline]828    pub fn from_bytes(bytes: impl TryInto<u64>) -> Size {829        let bytes: u64 = bytes.try_into().ok().unwrap();830        Size { raw: bytes }831    }832833    #[inline]834    pub fn bytes(self) -> u64 {835        self.raw836    }837838    #[inline]839    pub fn bytes_usize(self) -> usize {840        self.bytes().try_into().unwrap()841    }842843    #[inline]844    pub fn bits(self) -> u64 {845        #[cold]846        fn overflow(bytes: u64) -> ! {847            panic!("Size::bits: {bytes} bytes in bits doesn't fit in u64")848        }849850        self.bytes().checked_mul(8).unwrap_or_else(|| overflow(self.bytes()))851    }852853    #[inline]854    pub fn bits_usize(self) -> usize {855        self.bits().try_into().unwrap()856    }857858    #[inline]859    pub fn align_to(self, align: Align) -> Size {860        let mask = align.bytes() - 1;861        Size::from_bytes((self.bytes() + mask) & !mask)862    }863864    #[inline]865    pub fn is_aligned(self, align: Align) -> bool {866        let mask = align.bytes() - 1;867        self.bytes() & mask == 0868    }869870    #[inline]871    pub fn checked_add<C: HasDataLayout>(self, offset: Size, cx: &C) -> Option<Size> {872        let dl = cx.data_layout();873874        let bytes = self.bytes().checked_add(offset.bytes())?;875876        if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None }877    }878879    #[inline]880    pub fn checked_mul<C: HasDataLayout>(self, count: u64, cx: &C) -> Option<Size> {881        let dl = cx.data_layout();882883        let bytes = self.bytes().checked_mul(count)?;884        if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None }885    }886887    /// Truncates `value` to `self` bits and then sign-extends it to 128 bits888    /// (i.e., if it is negative, fill with 1's on the left).889    #[inline]890    pub fn sign_extend(self, value: u128) -> i128 {891        let size = self.bits();892        if size == 0 {893            // Truncated until nothing is left.894            return 0;895        }896        // Sign-extend it.897        let shift = 128 - size;898        // Shift the unsigned value to the left, then shift back to the right as signed899        // (essentially fills with sign bit on the left).900        ((value << shift) as i128) >> shift901    }902903    /// Truncates `value` to `self` bits.904    #[inline]905    pub fn truncate(self, value: u128) -> u128 {906        let size = self.bits();907        if size == 0 {908            // Truncated until nothing is left.909            return 0;910        }911        let shift = 128 - size;912        // Truncate (shift left to drop out leftover values, shift right to fill with zeroes).913        (value << shift) >> shift914    }915916    #[inline]917    pub fn signed_int_min(&self) -> i128 {918        self.sign_extend(1_u128 << (self.bits() - 1))919    }920921    #[inline]922    pub fn signed_int_max(&self) -> i128 {923        i128::MAX >> (128 - self.bits())924    }925926    #[inline]927    pub fn unsigned_int_max(&self) -> u128 {928        u128::MAX >> (128 - self.bits())929    }930}931932// Panicking addition, subtraction and multiplication for convenience.933// Avoid during layout computation, return `LayoutError` instead.934935impl Add for Size {936    type Output = Size;937    #[inline]938    fn add(self, other: Size) -> Size {939        Size::from_bytes(self.bytes().checked_add(other.bytes()).unwrap_or_else(|| {940            panic!("Size::add: {} + {} doesn't fit in u64", self.bytes(), other.bytes())941        }))942    }943}944945impl Sub for Size {946    type Output = Size;947    #[inline]948    fn sub(self, other: Size) -> Size {949        Size::from_bytes(self.bytes().checked_sub(other.bytes()).unwrap_or_else(|| {950            panic!("Size::sub: {} - {} would result in negative size", self.bytes(), other.bytes())951        }))952    }953}954955impl Mul<Size> for u64 {956    type Output = Size;957    #[inline]958    fn mul(self, size: Size) -> Size {959        size * self960    }961}962963impl Mul<u64> for Size {964    type Output = Size;965    #[inline]966    fn mul(self, count: u64) -> Size {967        match self.bytes().checked_mul(count) {968            Some(bytes) => Size::from_bytes(bytes),969            None => panic!("Size::mul: {} * {} doesn't fit in u64", self.bytes(), count),970        }971    }972}973974impl AddAssign for Size {975    #[inline]976    fn add_assign(&mut self, other: Size) {977        *self = *self + other;978    }979}980981#[cfg(feature = "nightly")]982impl Step for Size {983    #[inline]984    fn steps_between(start: &Self, end: &Self) -> (usize, Option<usize>) {985        u64::steps_between(&start.bytes(), &end.bytes())986    }987988    #[inline]989    fn forward_checked(start: Self, count: usize) -> Option<Self> {990        u64::forward_checked(start.bytes(), count).map(Self::from_bytes)991    }992993    #[inline]994    fn forward(start: Self, count: usize) -> Self {995        Self::from_bytes(u64::forward(start.bytes(), count))996    }997998    #[inline]999    unsafe fn forward_unchecked(start: Self, count: usize) -> Self {1000        Self::from_bytes(unsafe { u64::forward_unchecked(start.bytes(), count) })1001    }10021003    #[inline]1004    fn backward_checked(start: Self, count: usize) -> Option<Self> {1005        u64::backward_checked(start.bytes(), count).map(Self::from_bytes)1006    }10071008    #[inline]1009    fn backward(start: Self, count: usize) -> Self {1010        Self::from_bytes(u64::backward(start.bytes(), count))1011    }10121013    #[inline]1014    unsafe fn backward_unchecked(start: Self, count: usize) -> Self {1015        Self::from_bytes(unsafe { u64::backward_unchecked(start.bytes(), count) })1016    }1017}10181019/// Alignment of a type in bytes (always a power of two).1020#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]1021#[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]1022pub struct Align {1023    pow2: u8,1024}10251026// This is debug-printed a lot in larger structs, don't waste too much space there1027impl fmt::Debug for Align {1028    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {1029        write!(f, "Align({} bytes)", self.bytes())1030    }1031}10321033#[derive(Clone, Copy)]1034pub enum AlignFromBytesError {1035    NotPowerOfTwo(u64),1036    TooLarge(u64),1037}10381039impl fmt::Debug for AlignFromBytesError {1040    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {1041        fmt::Display::fmt(self, f)1042    }1043}10441045impl fmt::Display for AlignFromBytesError {1046    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {1047        match self {1048            AlignFromBytesError::NotPowerOfTwo(align) => write!(f, "{align} is not a power of 2"),1049            AlignFromBytesError::TooLarge(align) => write!(f, "{align} is too large"),1050        }1051    }1052}10531054impl Align {1055    pub const ONE: Align = Align { pow2: 0 };1056    pub const EIGHT: Align = Align { pow2: 3 };1057    // LLVM has a maximal supported alignment of 2^29, we inherit that.1058    pub const MAX: Align = Align { pow2: 29 };10591060    /// Either `1 << (pointer_bits - 1)` or [`Align::MAX`], whichever is smaller.1061    #[inline]1062    pub fn max_for_target(tdl: &TargetDataLayout) -> Align {1063        let pointer_bits = tdl.pointer_size().bits();1064        if let Ok(pointer_bits) = u8::try_from(pointer_bits)1065            && pointer_bits <= Align::MAX.pow21066        {1067            Align { pow2: pointer_bits - 1 }1068        } else {1069            Align::MAX1070        }1071    }10721073    #[inline]1074    pub fn from_bits(bits: u64) -> Result<Align, AlignFromBytesError> {1075        Align::from_bytes(Size::from_bits(bits).bytes())1076    }10771078    #[inline]1079    pub const fn from_bytes(align: u64) -> Result<Align, AlignFromBytesError> {1080        // Treat an alignment of 0 bytes like 1-byte alignment.1081        if align == 0 {1082            return Ok(Align::ONE);1083        }10841085        #[cold]1086        const fn not_power_of_2(align: u64) -> AlignFromBytesError {1087            AlignFromBytesError::NotPowerOfTwo(align)1088        }10891090        #[cold]1091        const fn too_large(align: u64) -> AlignFromBytesError {1092            AlignFromBytesError::TooLarge(align)1093        }10941095        let tz = align.trailing_zeros();1096        if align != (1 << tz) {1097            return Err(not_power_of_2(align));1098        }10991100        let pow2 = tz as u8;1101        if pow2 > Self::MAX.pow2 {1102            return Err(too_large(align));1103        }11041105        Ok(Align { pow2 })1106    }11071108    #[inline]1109    pub const fn bytes(self) -> u64 {1110        1 << self.pow21111    }11121113    #[inline]1114    pub fn bytes_usize(self) -> usize {1115        self.bytes().try_into().unwrap()1116    }11171118    #[inline]1119    pub const fn bits(self) -> u64 {1120        self.bytes() * 81121    }11221123    #[inline]1124    pub fn bits_usize(self) -> usize {1125        self.bits().try_into().unwrap()1126    }11271128    /// Obtain the greatest factor of `size` that is an alignment1129    /// (the largest power of two the Size is a multiple of).1130    ///1131    /// Note that all numbers are factors of 01132    #[inline]1133    pub fn max_aligned_factor(size: Size) -> Align {1134        Align { pow2: size.bytes().trailing_zeros() as u8 }1135    }11361137    /// Reduces Align to an aligned factor of `size`.1138    #[inline]1139    pub fn restrict_for_offset(self, size: Size) -> Align {1140        self.min(Align::max_aligned_factor(size))1141    }1142}11431144/// A pair of alignments, ABI-mandated and preferred.1145///1146/// The "preferred" alignment is an LLVM concept that is virtually meaningless to Rust code:1147/// it is not exposed semantically to programmers nor can they meaningfully affect it.1148/// The only concern for us is that preferred alignment must not be less than the mandated alignment1149/// and thus in practice the two values are almost always identical.1150///1151/// An example of a rare thing actually affected by preferred alignment is aligning of statics.1152/// It is of effectively no consequence for layout in structs and on the stack.1153#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]1154#[cfg_attr(feature = "nightly", derive(StableHash))]1155pub struct AbiAlign {1156    pub abi: Align,1157}11581159impl AbiAlign {1160    #[inline]1161    pub fn new(align: Align) -> AbiAlign {1162        AbiAlign { abi: align }1163    }11641165    #[inline]1166    pub fn min(self, other: AbiAlign) -> AbiAlign {1167        AbiAlign { abi: self.abi.min(other.abi) }1168    }11691170    #[inline]1171    pub fn max(self, other: AbiAlign) -> AbiAlign {1172        AbiAlign { abi: self.abi.max(other.abi) }1173    }1174}11751176impl Deref for AbiAlign {1177    type Target = Align;11781179    fn deref(&self) -> &Self::Target {1180        &self.abi1181    }1182}11831184/// Integers, also used for enum discriminants.1185#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]1186#[cfg_attr(feature = "nightly", derive(Encodable_NoContext, Decodable_NoContext, StableHash))]1187pub enum Integer {1188    I8,1189    I16,1190    I32,1191    I64,1192    I128,1193}11941195impl Integer {1196    pub fn int_ty_str(self) -> &'static str {1197        use Integer::*;1198        match self {1199            I8 => "i8",1200            I16 => "i16",1201            I32 => "i32",1202            I64 => "i64",1203            I128 => "i128",1204        }1205    }12061207    pub fn uint_ty_str(self) -> &'static str {1208        use Integer::*;1209        match self {1210            I8 => "u8",1211            I16 => "u16",1212            I32 => "u32",1213            I64 => "u64",1214            I128 => "u128",1215        }1216    }12171218    #[inline]1219    pub fn size(self) -> Size {1220        use Integer::*;1221        match self {1222            I8 => Size::from_bytes(1),1223            I16 => Size::from_bytes(2),1224            I32 => Size::from_bytes(4),1225            I64 => Size::from_bytes(8),1226            I128 => Size::from_bytes(16),1227        }1228    }12291230    /// Gets the Integer type from an IntegerType.1231    pub fn from_attr<C: HasDataLayout>(cx: &C, ity: IntegerType) -> Integer {1232        let dl = cx.data_layout();12331234        match ity {1235            IntegerType::Pointer(_) => dl.ptr_sized_integer(),1236            IntegerType::Fixed(x, _) => x,1237        }1238    }12391240    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAlign {1241        use Integer::*;1242        let dl = cx.data_layout();12431244        AbiAlign::new(match self {1245            I8 => dl.i8_align,1246            I16 => dl.i16_align,1247            I32 => dl.i32_align,1248            I64 => dl.i64_align,1249            I128 => dl.i128_align,1250        })1251    }12521253    /// Returns the largest signed value that can be represented by this Integer.1254    #[inline]1255    pub fn signed_max(self) -> i128 {1256        use Integer::*;1257        match self {1258            I8 => i8::MAX as i128,1259            I16 => i16::MAX as i128,1260            I32 => i32::MAX as i128,1261            I64 => i64::MAX as i128,1262            I128 => i128::MAX,1263        }1264    }12651266    /// Returns the smallest signed value that can be represented by this Integer.1267    #[inline]1268    pub fn signed_min(self) -> i128 {1269        use Integer::*;1270        match self {1271            I8 => i8::MIN as i128,1272            I16 => i16::MIN as i128,1273            I32 => i32::MIN as i128,1274            I64 => i64::MIN as i128,1275            I128 => i128::MIN,1276        }1277    }12781279    /// Finds the smallest Integer type which can represent the signed value.1280    #[inline]1281    pub fn fit_signed(x: i128) -> Integer {1282        use Integer::*;1283        match x {1284            -0x0000_0000_0000_0080..=0x0000_0000_0000_007f => I8,1285            -0x0000_0000_0000_8000..=0x0000_0000_0000_7fff => I16,1286            -0x0000_0000_8000_0000..=0x0000_0000_7fff_ffff => I32,1287            -0x8000_0000_0000_0000..=0x7fff_ffff_ffff_ffff => I64,1288            _ => I128,1289        }1290    }12911292    /// Finds the smallest Integer type which can represent the unsigned value.1293    #[inline]1294    pub fn fit_unsigned(x: u128) -> Integer {1295        use Integer::*;1296        match x {1297            0..=0x0000_0000_0000_00ff => I8,1298            0..=0x0000_0000_0000_ffff => I16,1299            0..=0x0000_0000_ffff_ffff => I32,1300            0..=0xffff_ffff_ffff_ffff => I64,1301            _ => I128,1302        }1303    }13041305    /// Finds the smallest integer with the given alignment.1306    pub fn for_align<C: HasDataLayout>(cx: &C, wanted: Align) -> Option<Integer> {1307        use Integer::*;1308        let dl = cx.data_layout();13091310        [I8, I16, I32, I64, I128].into_iter().find(|&candidate| {1311            wanted == candidate.align(dl).abi && wanted.bytes() == candidate.size().bytes()1312        })1313    }13141315    /// Find the largest integer with the given alignment or less.1316    pub fn approximate_align<C: HasDataLayout>(cx: &C, wanted: Align) -> Integer {1317        use Integer::*;1318        let dl = cx.data_layout();13191320        // FIXME(eddyb) maybe include I128 in the future, when it works everywhere.1321        for candidate in [I64, I32, I16] {1322            if wanted >= candidate.align(dl).abi && wanted.bytes() >= candidate.size().bytes() {1323                return candidate;1324            }1325        }1326        I81327    }13281329    // FIXME(eddyb) consolidate this and other methods that find the appropriate1330    // `Integer` given some requirements.1331    #[inline]1332    pub fn from_size(size: Size) -> Result<Self, String> {1333        match size.bits() {1334            8 => Ok(Integer::I8),1335            16 => Ok(Integer::I16),1336            32 => Ok(Integer::I32),1337            64 => Ok(Integer::I64),1338            128 => Ok(Integer::I128),1339            _ => Err(format!("rust does not support integers with {} bits", size.bits())),1340        }1341    }1342}13431344/// Floating-point types.1345#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]1346#[cfg_attr(feature = "nightly", derive(StableHash))]1347pub enum Float {1348    F16,1349    F32,1350    F64,1351    F128,1352}13531354impl Float {1355    pub fn size(self) -> Size {1356        use Float::*;13571358        match self {1359            F16 => Size::from_bits(16),1360            F32 => Size::from_bits(32),1361            F64 => Size::from_bits(64),1362            F128 => Size::from_bits(128),1363        }1364    }13651366    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAlign {1367        use Float::*;1368        let dl = cx.data_layout();13691370        AbiAlign::new(match self {1371            F16 => dl.f16_align,1372            F32 => dl.f32_align,1373            F64 => dl.f64_align,1374            F128 => dl.f128_align,1375        })1376    }1377}13781379/// Fundamental unit of memory access and layout.1380#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]1381#[cfg_attr(feature = "nightly", derive(StableHash))]1382pub enum Primitive {1383    /// The `bool` is the signedness of the `Integer` type.1384    ///1385    /// One would think we would not care about such details this low down,1386    /// but some ABIs are described in terms of C types and ISAs where the1387    /// integer arithmetic is done on {sign,zero}-extended registers, e.g.1388    /// a negative integer passed by zero-extension will appear positive in1389    /// the callee, and most operations on it will produce the wrong values.1390    Int(Integer, bool),1391    Float(Float),1392    Pointer(AddressSpace),1393}13941395impl Primitive {1396    pub fn size<C: HasDataLayout>(self, cx: &C) -> Size {1397        use Primitive::*;1398        let dl = cx.data_layout();13991400        match self {1401            Int(i, _) => i.size(),1402            Float(f) => f.size(),1403            Pointer(a) => dl.pointer_size_in(a),1404        }1405    }14061407    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAlign {1408        use Primitive::*;1409        let dl = cx.data_layout();14101411        match self {1412            Int(i, _) => i.align(dl),1413            Float(f) => f.align(dl),1414            Pointer(a) => dl.pointer_align_in(a),1415        }1416    }1417}14181419/// Inclusive wrap-around range of valid values, that is, if1420/// start > end, it represents `start..=MAX`, followed by `0..=end`.1421///1422/// That is, for an i8 primitive, a range of `254..=2` means following1423/// sequence:1424///1425///    254 (-2), 255 (-1), 0, 1, 21426///1427/// This is intended specifically to mirror LLVM’s `!range` metadata semantics.1428#[derive(Clone, Copy, PartialEq, Eq, Hash)]1429#[cfg_attr(feature = "nightly", derive(StableHash))]1430pub struct WrappingRange {1431    pub start: u128,1432    pub end: u128,1433}14341435impl WrappingRange {1436    pub fn full(size: Size) -> Self {1437        Self { start: 0, end: size.unsigned_int_max() }1438    }14391440    /// Returns `true` if `v` is contained in the range.1441    #[inline(always)]1442    pub fn contains(&self, v: u128) -> bool {1443        if self.start <= self.end {1444            self.start <= v && v <= self.end1445        } else {1446            self.start <= v || v <= self.end1447        }1448    }14491450    /// Returns `true` if all the values in `other` are contained in this range,1451    /// when the values are considered as having width `size`.1452    #[inline(always)]1453    pub fn contains_range(&self, other: Self, size: Size) -> bool {1454        if self.is_full_for(size) {1455            true1456        } else {1457            let trunc = |x| size.truncate(x);14581459            let delta = self.start;1460            let max = trunc(self.end.wrapping_sub(delta));14611462            let other_start = trunc(other.start.wrapping_sub(delta));1463            let other_end = trunc(other.end.wrapping_sub(delta));14641465            // Having shifted both input ranges by `delta`, now we only need to check1466            // whether `0..=max` contains `other_start..=other_end`, which can only1467            // happen if the other doesn't wrap since `self` isn't everything.1468            (other_start <= other_end) && (other_end <= max)1469        }1470    }14711472    /// Returns `self` with replaced `start`1473    #[inline(always)]1474    fn with_start(mut self, start: u128) -> Self {1475        self.start = start;1476        self1477    }14781479    /// Returns `self` with replaced `end`1480    #[inline(always)]1481    fn with_end(mut self, end: u128) -> Self {1482        self.end = end;1483        self1484    }14851486    /// Returns `true` if `size` completely fills the range.1487    ///1488    /// Note that this is *not* the same as `self == WrappingRange::full(size)`.1489    /// Niche calculations can produce full ranges which are not the canonical one;1490    /// for example `Option<NonZero<u16>>` gets `valid_range: (..=0) | (1..)`.1491    #[inline]1492    fn is_full_for(&self, size: Size) -> bool {1493        let max_value = size.unsigned_int_max();1494        debug_assert!(self.start <= max_value && self.end <= max_value);1495        self.start == (self.end.wrapping_add(1) & max_value)1496    }14971498    /// Checks whether this range is considered non-wrapping when the values are1499    /// interpreted as *unsigned* numbers of width `size`.1500    ///1501    /// Returns `Ok(true)` if there's no wrap-around, `Ok(false)` if there is,1502    /// and `Err(..)` if the range is full so it depends how you think about it.1503    #[inline]1504    pub fn no_unsigned_wraparound(&self, size: Size) -> Result<bool, RangeFull> {1505        if self.is_full_for(size) { Err(..) } else { Ok(self.start <= self.end) }1506    }15071508    /// Checks whether this range is considered non-wrapping when the values are1509    /// interpreted as *signed* numbers of width `size`.1510    ///1511    /// This is heavily dependent on the `size`, as `100..=200` does wrap when1512    /// interpreted as `i8`, but doesn't when interpreted as `i16`.1513    ///1514    /// Returns `Ok(true)` if there's no wrap-around, `Ok(false)` if there is,1515    /// and `Err(..)` if the range is full so it depends how you think about it.1516    #[inline]1517    pub fn no_signed_wraparound(&self, size: Size) -> Result<bool, RangeFull> {1518        if self.is_full_for(size) {1519            Err(..)1520        } else {1521            let start: i128 = size.sign_extend(self.start);1522            let end: i128 = size.sign_extend(self.end);1523            Ok(start <= end)1524        }1525    }1526}15271528impl fmt::Debug for WrappingRange {1529    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {1530        if self.start > self.end {1531            write!(fmt, "(..={}) | ({}..)", self.end, self.start)?;1532        } else {1533            write!(fmt, "{}..={}", self.start, self.end)?;1534        }1535        Ok(())1536    }1537}15381539/// Information about one scalar component of a Rust type.1540#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]1541#[cfg_attr(feature = "nightly", derive(StableHash))]1542pub enum Scalar {1543    Initialized {1544        value: Primitive,15451546        // FIXME(eddyb) always use the shortest range, e.g., by finding1547        // the largest space between two consecutive valid values and1548        // taking everything else as the (shortest) valid range.1549        valid_range: WrappingRange,1550    },1551    Union {1552        /// Even for unions, we need to use the correct registers for the kind of1553        /// values inside the union, so we keep the `Primitive` type around. We1554        /// also use it to compute the size of the scalar.1555        /// However, unions never have niches and even allow undef,1556        /// so there is no `valid_range`.1557        value: Primitive,1558    },1559}15601561impl Scalar {1562    #[inline]1563    pub fn is_bool(&self) -> bool {1564        use Integer::*;1565        matches!(1566            self,1567            Scalar::Initialized {1568                value: Primitive::Int(I8, false),1569                valid_range: WrappingRange { start: 0, end: 1 }1570            }1571        )1572    }15731574    /// Get the primitive representation of this type, ignoring the valid range and whether the1575    /// value is allowed to be undefined (due to being a union).1576    pub fn primitive(&self) -> Primitive {1577        match *self {1578            Scalar::Initialized { value, .. } | Scalar::Union { value } => value,1579        }1580    }15811582    pub fn align(self, cx: &impl HasDataLayout) -> AbiAlign {1583        self.primitive().align(cx)1584    }15851586    pub fn size(self, cx: &impl HasDataLayout) -> Size {1587        self.primitive().size(cx)1588    }15891590    #[inline]1591    pub fn to_union(&self) -> Self {1592        Self::Union { value: self.primitive() }1593    }15941595    #[inline]1596    pub fn valid_range(&self, cx: &impl HasDataLayout) -> WrappingRange {1597        match *self {1598            Scalar::Initialized { valid_range, .. } => valid_range,1599            Scalar::Union { value } => WrappingRange::full(value.size(cx)),1600        }1601    }16021603    #[inline]1604    /// Allows the caller to mutate the valid range. This operation will panic if attempted on a1605    /// union.1606    pub fn valid_range_mut(&mut self) -> &mut WrappingRange {1607        match self {1608            Scalar::Initialized { valid_range, .. } => valid_range,1609            Scalar::Union { .. } => panic!("cannot change the valid range of a union"),1610        }1611    }16121613    /// Returns `true` if all possible numbers are valid, i.e `valid_range` covers the whole1614    /// layout.1615    #[inline]1616    pub fn is_always_valid<C: HasDataLayout>(&self, cx: &C) -> bool {1617        match *self {1618            Scalar::Initialized { valid_range, .. } => valid_range.is_full_for(self.size(cx)),1619            Scalar::Union { .. } => true,1620        }1621    }16221623    /// Returns `true` if this type can be left uninit.1624    #[inline]1625    pub fn is_uninit_valid(&self) -> bool {1626        match *self {1627            Scalar::Initialized { .. } => false,1628            Scalar::Union { .. } => true,1629        }1630    }16311632    /// Returns `true` if this is a signed integer scalar1633    #[inline]1634    pub fn is_signed(&self) -> bool {1635        match self.primitive() {1636            Primitive::Int(_, signed) => signed,1637            _ => false,1638        }1639    }1640}16411642// NOTE: This struct is generic over the FieldIdx for rust-analyzer usage.1643/// Describes how the fields of a type are located in memory.1644#[derive(PartialEq, Eq, Hash, Clone, Debug)]1645#[cfg_attr(feature = "nightly", derive(StableHash))]1646pub enum FieldsShape<FieldIdx: Idx> {1647    /// Scalar primitives and `!`, which never have fields.1648    Primitive,16491650    /// All fields start at no offset. The `usize` is the field count.1651    Union(NonZeroUsize),16521653    /// Array/vector-like placement, with all fields of identical types.1654    Array { stride: Size, count: u64 },16551656    /// Struct-like placement, with precomputed offsets.1657    ///1658    /// Fields are guaranteed to not overlap, but note that gaps1659    /// before, between and after all the fields are NOT always1660    /// padding, and as such their contents may not be discarded.1661    /// For example, enum variants leave a gap at the start,1662    /// where the discriminant field in the enum layout goes.1663    Arbitrary {1664        /// Offsets for the first byte of each field,1665        /// ordered to match the source definition order.1666        /// This vector does not go in increasing order.1667        // FIXME(eddyb) use small vector optimization for the common case.1668        offsets: IndexVec<FieldIdx, Size>,16691670        /// Maps memory order field indices to source order indices,1671        /// depending on how the fields were reordered (if at all).1672        /// This is a permutation, with both the source order and the1673        /// memory order using the same (0..n) index ranges.1674        ///1675        // FIXME(eddyb) build a better abstraction for permutations, if possible.1676        // FIXME(camlorn) also consider small vector optimization here.1677        in_memory_order: IndexVec<u32, FieldIdx>,1678    },1679}16801681impl<FieldIdx: Idx> FieldsShape<FieldIdx> {1682    #[inline]1683    pub fn count(&self) -> usize {1684        match *self {1685            FieldsShape::Primitive => 0,1686            FieldsShape::Union(count) => count.get(),1687            FieldsShape::Array { count, .. } => count.try_into().unwrap(),1688            FieldsShape::Arbitrary { ref offsets, .. } => offsets.len(),1689        }1690    }16911692    #[inline]1693    pub fn offset(&self, i: usize) -> Size {1694        match *self {1695            FieldsShape::Primitive => {1696                unreachable!("FieldsShape::offset: `Primitive`s have no fields")1697            }1698            FieldsShape::Union(count) => {1699                assert!(i < count.get(), "tried to access field {i} of union with {count} fields");1700                Size::ZERO1701            }1702            FieldsShape::Array { stride, count } => {1703                let i = u64::try_from(i).unwrap();1704                assert!(i < count, "tried to access field {i} of array with {count} fields");1705                stride * i1706            }1707            FieldsShape::Arbitrary { ref offsets, .. } => offsets[FieldIdx::new(i)],1708        }1709    }17101711    /// Gets source indices of the fields by increasing offsets.1712    #[inline]1713    pub fn index_by_increasing_offset(&self) -> impl ExactSizeIterator<Item = usize> {1714        // Primitives don't really have fields in the way that structs do,1715        // but having this return an empty iterator for them is unhelpful1716        // since that makes them look kinda like ZSTs, which they're not.1717        let pseudofield_count = if let FieldsShape::Primitive = self { 1 } else { self.count() };17181719        (0..pseudofield_count).map(move |i| match self {1720            FieldsShape::Primitive | FieldsShape::Union(_) | FieldsShape::Array { .. } => i,1721            FieldsShape::Arbitrary { in_memory_order, .. } => in_memory_order[i as u32].index(),1722        })1723    }1724}17251726/// An identifier that specifies the address space that some operation1727/// should operate on. Special address spaces have an effect on code generation,1728/// depending on the target and the address spaces it implements.1729#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]1730#[cfg_attr(feature = "nightly", derive(StableHash))]1731pub struct AddressSpace(pub u32);17321733impl AddressSpace {1734    /// LLVM's `0` address space.1735    pub const ZERO: Self = AddressSpace(0);1736    /// The address space for workgroup memory on nvptx and amdgpu.1737    /// See e.g. the `gpu_launch_sized_workgroup_mem` intrinsic for details.1738    pub const GPU_WORKGROUP: Self = AddressSpace(3);1739}17401741/// How many scalable vectors are in a `BackendRepr::ScalableVector`?1742#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]1743#[cfg_attr(feature = "nightly", derive(StableHash))]1744pub struct NumScalableVectors(pub u8);17451746impl NumScalableVectors {1747    /// Returns a `NumScalableVector` for a non-tuple scalable vector (e.g. a single vector).1748    pub fn for_non_tuple() -> Self {1749        NumScalableVectors(1)1750    }17511752    // Returns `NumScalableVectors` for values of two through eight, which are a valid number of1753    // fields for a tuple of scalable vectors to have. `1` is a valid value of `NumScalableVectors`1754    // but not for a tuple which would have a field count.1755    pub fn from_field_count(count: usize) -> Option<Self> {1756        match count {1757            2..8 => Some(NumScalableVectors(count as u8)),1758            _ => None,1759        }1760    }1761}17621763#[cfg(feature = "nightly")]1764impl IntoDiagArg for NumScalableVectors {1765    fn into_diag_arg(self, _: &mut Option<std::path::PathBuf>) -> DiagArgValue {1766        DiagArgValue::Str(std::borrow::Cow::Borrowed(match self.0 {1767            0 => panic!("`NumScalableVectors(0)` is illformed"),1768            1 => "one",1769            2 => "two",1770            3 => "three",1771            4 => "four",1772            5 => "five",1773            6 => "six",1774            7 => "seven",1775            8 => "eight",1776            _ => panic!("`NumScalableVectors(N)` for N>8 is illformed"),1777        }))1778    }1779}17801781/// The way we represent values to the backend1782///1783/// Previously this was conflated with the "ABI" a type is given, as in the platform-specific ABI.1784/// In reality, this implies little about that, but is mostly used to describe the syntactic form1785/// emitted for the backend, as most backends handle SSA values and blobs of memory differently.1786/// The psABI may need consideration in doing so, but this enum does not constitute a promise for1787/// how the value will be lowered to the calling convention, in itself.1788///1789/// Generally, a codegen backend will prefer to handle smaller values as a scalar or short vector,1790/// and larger values will usually prefer to be represented as memory.1791#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]1792#[cfg_attr(feature = "nightly", derive(StableHash))]1793pub enum BackendRepr {1794    Scalar(Scalar),1795    ScalarPair(Scalar, Scalar),1796    SimdScalableVector {1797        element: Scalar,1798        count: u64,1799        number_of_vectors: NumScalableVectors,1800    },1801    SimdVector {1802        element: Scalar,1803        count: u64,1804    },1805    // FIXME: I sometimes use memory, sometimes use an IR aggregate!1806    Memory {1807        /// If true, the size is exact, otherwise it's only a lower bound.1808        sized: bool,1809    },1810}18111812impl BackendRepr {1813    /// Returns `true` if the layout corresponds to an unsized type.1814    #[inline]1815    pub fn is_unsized(&self) -> bool {1816        match *self {1817            BackendRepr::Scalar(_)1818            | BackendRepr::ScalarPair(..)1819            // FIXME(rustc_scalable_vector): Scalable vectors are `Sized` while the1820            // `sized_hierarchy` feature is not yet fully implemented. After `sized_hierarchy` is1821            // fully implemented, scalable vectors will remain `Sized`, they just won't be1822            // `const Sized` - whether `is_unsized` continues to return `false` at that point will1823            // need to be revisited and will depend on what `is_unsized` is used for.1824            | BackendRepr::SimdScalableVector { .. }1825            | BackendRepr::SimdVector { .. } => false,1826            BackendRepr::Memory { sized } => !sized,1827        }1828    }18291830    #[inline]1831    pub fn is_sized(&self) -> bool {1832        !self.is_unsized()1833    }18341835    /// Returns `true` if this is a single signed integer scalar.1836    /// Sanity check: panics if this is not a scalar type (see PR #70189).1837    #[inline]1838    pub fn is_signed(&self) -> bool {1839        match self {1840            BackendRepr::Scalar(scal) => scal.is_signed(),1841            _ => panic!("`is_signed` on non-scalar ABI {self:?}"),1842        }1843    }18441845    /// Returns `true` if this is a scalar type1846    #[inline]1847    pub fn is_scalar(&self) -> bool {1848        matches!(*self, BackendRepr::Scalar(_))1849    }18501851    /// Returns `true` if this is a bool1852    #[inline]1853    pub fn is_bool(&self) -> bool {1854        matches!(*self, BackendRepr::Scalar(s) if s.is_bool())1855    }18561857    /// The psABI alignment for a `Scalar` or `ScalarPair`1858    ///1859    /// `None` for other variants.1860    pub fn scalar_align<C: HasDataLayout>(&self, cx: &C) -> Option<Align> {1861        match *self {1862            BackendRepr::Scalar(s) => Some(s.align(cx).abi),1863            BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),1864            // The align of a Vector can vary in surprising ways1865            BackendRepr::SimdVector { .. }1866            | BackendRepr::Memory { .. }1867            | BackendRepr::SimdScalableVector { .. } => None,1868        }1869    }18701871    /// The psABI size for a `Scalar` or `ScalarPair`1872    ///1873    /// `None` for other variants1874    pub fn scalar_size<C: HasDataLayout>(&self, cx: &C) -> Option<Size> {1875        match *self {1876            // No padding in scalars.1877            BackendRepr::Scalar(s) => Some(s.size(cx)),1878            // May have some padding between the pair.1879            BackendRepr::ScalarPair(s1, s2) => {1880                let field2_offset = s1.size(cx).align_to(s2.align(cx).abi);1881                let size = (field2_offset + s2.size(cx)).align_to(1882                    self.scalar_align(cx)1883                        // We absolutely must have an answer here or everything is FUBAR.1884                        .unwrap(),1885                );1886                Some(size)1887            }1888            // The size of a Vector can vary in surprising ways1889            BackendRepr::SimdVector { .. }1890            | BackendRepr::Memory { .. }1891            | BackendRepr::SimdScalableVector { .. } => None,1892        }1893    }18941895    /// Discard validity range information and allow undef.1896    pub fn to_union(&self) -> Self {1897        match *self {1898            BackendRepr::Scalar(s) => BackendRepr::Scalar(s.to_union()),1899            BackendRepr::ScalarPair(s1, s2) => {1900                BackendRepr::ScalarPair(s1.to_union(), s2.to_union())1901            }1902            BackendRepr::SimdVector { element, count } => {1903                BackendRepr::SimdVector { element: element.to_union(), count }1904            }1905            BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },1906            BackendRepr::SimdScalableVector { element, count, number_of_vectors } => {1907                BackendRepr::SimdScalableVector {1908                    element: element.to_union(),1909                    count,1910                    number_of_vectors,1911                }1912            }1913        }1914    }19151916    pub fn eq_up_to_validity(&self, other: &Self) -> bool {1917        match (self, other) {1918            // Scalar, Vector, ScalarPair have `Scalar` in them where we ignore validity ranges.1919            // We do *not* ignore the sign since it matters for some ABIs (e.g. s390x).1920            (BackendRepr::Scalar(l), BackendRepr::Scalar(r)) => l.primitive() == r.primitive(),1921            (1922                BackendRepr::SimdVector { element: element_l, count: count_l },1923                BackendRepr::SimdVector { element: element_r, count: count_r },1924            ) => element_l.primitive() == element_r.primitive() && count_l == count_r,1925            (BackendRepr::ScalarPair(l1, l2), BackendRepr::ScalarPair(r1, r2)) => {1926                l1.primitive() == r1.primitive() && l2.primitive() == r2.primitive()1927            }1928            // Everything else must be strictly identical.1929            _ => self == other,1930        }1931    }1932}19331934// NOTE: This struct is generic over the FieldIdx and VariantIdx for rust-analyzer usage.1935#[derive(PartialEq, Eq, Hash, Clone, Debug)]1936#[cfg_attr(feature = "nightly", derive(StableHash))]1937pub enum Variants<FieldIdx: Idx, VariantIdx: Idx> {1938    /// A type with no valid variants. Must be uninhabited.1939    Empty,19401941    /// Single enum variants, structs/tuples, unions, and all non-ADTs.1942    Single {1943        /// Always `0` for types that cannot have multiple variants.1944        index: VariantIdx,1945    },19461947    /// Enum-likes with more than one variant: each variant comes with1948    /// a *discriminant* (usually the same as the variant index but the user can1949    /// assign explicit discriminant values). That discriminant is encoded1950    /// as a *tag* on the machine. The layout of each variant is1951    /// a struct, and they all have space reserved for the tag.1952    /// For enums, the tag is the sole field of the layout.1953    Multiple {1954        tag: Scalar,1955        tag_encoding: TagEncoding<VariantIdx>,1956        tag_field: FieldIdx,1957        variants: IndexVec<VariantIdx, VariantLayout<FieldIdx>>,1958    },1959}19601961// NOTE: This struct is generic over the VariantIdx for rust-analyzer usage.1962#[derive(PartialEq, Eq, Hash, Copy, Clone, Debug)]1963#[cfg_attr(feature = "nightly", derive(StableHash))]1964pub enum TagEncoding<VariantIdx: Idx> {1965    /// The tag directly stores the discriminant, but possibly with a smaller layout1966    /// (so converting the tag to the discriminant can require sign extension).1967    Direct,19681969    /// Niche (values invalid for a type) encoding the discriminant.1970    /// Note that for this encoding, the discriminant and variant index of each variant coincide!1971    /// This invariant is codified as part of [`layout_sanity_check`](../rustc_ty_utils/layout/invariant/fn.layout_sanity_check.html).1972    ///1973    /// The variant `untagged_variant` contains a niche at an arbitrary1974    /// offset (field [`Variants::Multiple::tag_field`] of the enum).1975    /// For a variant with variant index `i`, such that `i != untagged_variant`,1976    /// the tag is set to `(i - niche_variants.start).wrapping_add(niche_start)`1977    /// (this is wrapping arithmetic using the type of the niche field, cf. the1978    /// [`tag_for_variant`](../rustc_const_eval/interpret/struct.InterpCx.html#method.tag_for_variant)1979    /// query implementation).1980    /// To recover the variant index `i` from a `tag`, the above formula has to be reversed,1981    /// i.e. `i = tag.wrapping_sub(niche_start) + niche_variants.start`. If `i` ends up outside1982    /// `niche_variants`, the tag must have encoded the `untagged_variant`.1983    ///1984    /// For example, `Option<(usize, &T)>`  is represented such that the tag for1985    /// `None` is the null pointer in the second tuple field, and1986    /// `Some` is the identity function (with a non-null reference)1987    /// and has no additional tag, i.e. the reference being non-null uniquely identifies this variant.1988    ///1989    /// Other variants that are not `untagged_variant` and that are outside the `niche_variants`1990    /// range cannot be represented; they must be uninhabited.1991    /// Nonetheless, uninhabited variants can also fall into the range of `niche_variants`.1992    Niche {1993        untagged_variant: VariantIdx,1994        /// This range *may* contain `untagged_variant` or uninhabited variants;1995        /// these are then just "dead values" and not used to encode anything.1996        niche_variants: RangeInclusive<VariantIdx>,1997        /// This is inbounds of the type of the niche field1998        /// (not sign-extended, i.e., all bits beyond the niche field size are 0).1999        niche_start: u128,2000    },