1use std::fmt::{self, Write};2use std::num::NonZero;3use std::ops::Deref;4use std::range::{RangeFrom, RangeInclusive, RangeToInclusive};5use std::{cmp, iter};67use rustc_hashes::Hash64;8use rustc_index::Idx;9use rustc_index::bit_set::BitMatrix;10use tracing::{debug, trace};1112use crate::{13    AbiAlign, Align, BackendLaneCount, BackendRepr, FieldsShape, HasDataLayout, IndexSlice,14    IndexVec, Integer, LayoutData, Niche, NicheOptimizations, NumScalableVectors, Primitive,15    ReprOptions, Scalar, Size, StructKind, TagEncoding, TargetDataLayout, VariantLayout, Variants,16    WrappingRange,17};1819mod coroutine;20mod simple;2122#[cfg(feature = "nightly")]23mod ty;2425#[cfg(feature = "nightly")]26pub use ty::{Layout, TyAbiInterface, TyAndLayout};2728rustc_index::newtype_index! {29    /// The *source-order* index of a field in a variant.30    ///31    /// This is how most code after type checking refers to fields, rather than32    /// using names (as names have hygiene complications and more complex lookup).33    ///34    /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order.35    /// (It is for `repr(C)` `struct`s, however.)36    ///37    /// For example, in the following types,38    /// ```rust39    /// # enum Never {}40    /// # #[repr(u16)]41    /// enum Demo1 {42    ///    Variant0 { a: Never, b: i32 } = 100,43    ///    Variant1 { c: u8, d: u64 } = 10,44    /// }45    /// struct Demo2 { e: u8, f: u16, g: u8 }46    /// ```47    /// `b` is `FieldIdx(1)` in `VariantIdx(0)`,48    /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and49    /// `f` is `FieldIdx(1)` in `VariantIdx(0)`.50    #[stable_hash]51    #[encodable]52    #[orderable]53    #[gate_rustc_only]54    pub struct FieldIdx {}55}5657impl FieldIdx {58    /// The second field, at index 1.59    ///60    /// For use alongside [`FieldIdx::ZERO`], particularly with scalar pairs.61    pub const ONE: FieldIdx = FieldIdx::from_u32(1);62}6364rustc_index::newtype_index! {65    /// The *source-order* index of a variant in a type.66    ///67    /// For enums, these are always `0..variant_count`, regardless of any68    /// custom discriminants that may have been defined, and including any69    /// variants that may end up uninhabited due to field types.  (Some of the70    /// variants may not be present in a monomorphized ABI [`Variants`], but71    /// those skipped variants are always counted when determining the *index*.)72    ///73    /// `struct`s, `tuples`, and `unions`s are considered to have a single variant74    /// with variant index zero, aka [`FIRST_VARIANT`].75    #[stable_hash]76    #[encodable]77    #[orderable]78    #[gate_rustc_only]79    pub struct VariantIdx {80        /// Equivalent to `VariantIdx(0)`.81        const FIRST_VARIANT = 0;82    }83}8485// A variant is absent if it's uninhabited and only has ZST fields.86// Present uninhabited variants only require space for their fields,87// but *not* an encoding of the discriminant (e.g., a tag value).88// See issue #49298 for more details on the need to leave space89// for non-ZST uninhabited data (mostly partial initialization).90fn absent<'a, FieldIdx, VariantIdx, F>(fields: &IndexSlice<FieldIdx, F>) -> bool91where92    FieldIdx: Idx,93    VariantIdx: Idx,94    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,95{96    let uninhabited = fields.iter().any(|f| f.is_uninhabited());97    // We cannot ignore alignment; that might lead us to entirely discard a variant and98    // produce an enum that is less aligned than it should be!99    let is_1zst = fields.iter().all(|f| f.is_1zst());100    uninhabited && is_1zst101}102103/// Determines towards which end of a struct layout optimizations will try to place the best niches.104enum NicheBias {105    Start,106    End,107}108109#[derive(Copy, Clone, Debug, PartialEq, Eq)]110pub enum LayoutCalculatorError<F> {111    /// An unsized type was found in a location where a sized type was expected.112    ///113    /// This is not always a compile error, for example if there is a `[T]: Sized`114    /// bound in a where clause.115    ///116    /// Contains the field that was unexpectedly unsized.117    UnexpectedUnsized(F),118119    /// A type was too large for the target platform.120    SizeOverflow,121122    /// A union had no fields.123    EmptyUnion,124125    /// The fields or variants have irreconcilable reprs126    ReprConflict,127128    /// The length of an SIMD type is zero129    ZeroLengthSimdType,130131    /// The length of an SIMD type exceeds the maximum number of lanes132    OversizedSimdType { max_lanes: usize },133134    /// An element type of an SIMD type isn't a primitive135    NonPrimitiveSimdType(F),136}137138impl<F> LayoutCalculatorError<F> {139    pub fn without_payload(&self) -> LayoutCalculatorError<()> {140        use LayoutCalculatorError::*;141        match *self {142            UnexpectedUnsized(_) => UnexpectedUnsized(()),143            SizeOverflow => SizeOverflow,144            EmptyUnion => EmptyUnion,145            ReprConflict => ReprConflict,146            ZeroLengthSimdType => ZeroLengthSimdType,147            OversizedSimdType { max_lanes } => OversizedSimdType { max_lanes },148            NonPrimitiveSimdType(_) => NonPrimitiveSimdType(()),149        }150    }151152    /// Format an untranslated diagnostic for this type153    ///154    /// Intended for use by rust-analyzer, as neither it nor `rustc_abi` depend on fluent infra.155    pub fn fallback_fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {156        use LayoutCalculatorError::*;157        f.write_str(match self {158            UnexpectedUnsized(_) => "an unsized type was found where a sized type was expected",159            SizeOverflow => "size overflow",160            EmptyUnion => "type is a union with no fields",161            ReprConflict => "type has an invalid repr",162            ZeroLengthSimdType | OversizedSimdType { .. } | NonPrimitiveSimdType(_) => {163                "invalid simd type definition"164            }165        })166    }167}168169type LayoutCalculatorResult<FieldIdx, VariantIdx, F> =170    Result<LayoutData<FieldIdx, VariantIdx>, LayoutCalculatorError<F>>;171172#[derive(Clone, Copy, Debug)]173pub struct LayoutCalculator<Cx> {174    pub cx: Cx,175}176177impl<Cx: HasDataLayout> LayoutCalculator<Cx> {178    pub fn new(cx: Cx) -> Self {179        Self { cx }180    }181182    pub fn layout_of_array_like<FieldIdx: Idx, VariantIdx: Idx, F>(183        &self,184        element: &LayoutData<FieldIdx, VariantIdx>,185        count_if_sized: Option<u64>, // None for slices186    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {187        let count = count_if_sized.unwrap_or(0);188        let size =189            element.size.checked_mul(count, &self.cx).ok_or(LayoutCalculatorError::SizeOverflow)?;190191        Ok(LayoutData {192            variants: Variants::Single { index: VariantIdx::new(0) },193            fields: FieldsShape::Array { stride: element.size, count },194            backend_repr: BackendRepr::Memory { sized: count_if_sized.is_some() },195            largest_niche: element.largest_niche.filter(|_| count != 0),196            uninhabited: element.uninhabited && count != 0,197            align: element.align,198            size,199            max_repr_align: None,200            unadjusted_abi_align: element.align.abi,201            repr_c: element.repr_c,202            randomization_seed: element.randomization_seed.wrapping_add(Hash64::new(count)),203        })204    }205206    pub fn layout_of_scalable_vector_type<FieldIdx, VariantIdx, F>(207        &self,208        element: F,209        count: u64,210        number_of_vectors: NumScalableVectors,211    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>212    where213        FieldIdx: Idx,214        VariantIdx: Idx,215        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,216    {217        vector_type_layout(218            SimdVectorKind::Scalable(number_of_vectors),219            self.cx.data_layout(),220            element,221            count,222        )223    }224225    pub fn layout_of_simd_type<FieldIdx, VariantIdx, F>(226        &self,227        element: F,228        count: u64,229        repr_packed: bool,230    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>231    where232        FieldIdx: Idx,233        VariantIdx: Idx,234        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,235    {236        let kind = if repr_packed { SimdVectorKind::PackedFixed } else { SimdVectorKind::Fixed };237        vector_type_layout(kind, self.cx.data_layout(), element, count)238    }239240    /// Compute the layout for a coroutine.241    ///242    /// This uses dedicated code instead of [`Self::layout_of_struct_or_enum`], as coroutine243    /// fields may be shared between multiple variants (see the [`coroutine`] module for details).244    pub fn layout_of_coroutine<'a, F, VariantIdx, FieldIdx, LocalIdx>(245        &self,246        local_layouts: &IndexSlice<LocalIdx, F>,247        prefix_layouts: IndexVec<FieldIdx, F>,248        variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,249        storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,250        tag_to_layout: impl Fn(Scalar) -> F,251    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>252    where253        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,254        VariantIdx: Idx,255        FieldIdx: Idx,256        LocalIdx: Idx,257    {258        coroutine::layout(259            self,260            local_layouts,261            prefix_layouts,262            variant_fields,263            storage_conflicts,264            tag_to_layout,265        )266    }267268    /// Compute the layout for a univariant (see [`Variants::Single`]).269    ///270    /// As a consumer of `rustc_abi`, you should only use this method for non-ADTs.271    /// For structs and univariant enums, use [`Self::layout_of_struct`] instead272    /// (it uses this function internally).273    pub fn layout_of_univariant<'a, FieldIdx, VariantIdx, F>(274        &self,275        fields: &IndexSlice<FieldIdx, F>,276        repr: &ReprOptions,277        kind: StructKind,278    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>279    where280        FieldIdx: Idx,281        VariantIdx: Idx,282        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,283    {284        let dl = self.cx.data_layout();285        let layout = self.layout_of_univariant_biased(fields, repr, kind, NicheBias::Start);286        // Enums prefer niches close to the beginning or the end of the variants so that other287        // (smaller) data-carrying variants can be packed into the space after/before the niche.288        // If the default field ordering does not give us a niche at the front then we do a second289        // run and bias niches to the right and then check which one is closer to one of the290        // struct's edges.291        if let Ok(layout) = &layout292            // Don't try to calculate an end-biased layout for unsizable structs,293            // otherwise we could end up with different layouts for294            // Foo<Type> and Foo<dyn Trait> which would break unsizing.295            && !matches!(kind, StructKind::MaybeUnsized)296            && let Some(niche) = layout.largest_niche297            && let head_space = niche.offset.bytes()298            && let niche_len = niche.value.size(dl).bytes()299            && let tail_space = layout.size.bytes() - head_space - niche_len300            // This may end up doing redundant work if the niche is already in the last301            // field (e.g. a trailing bool) and there is tail padding. But it's non-trivial302            // to get the unpadded size so we try anyway.303            && (fields.len() > 1 && head_space != 0 && tail_space > 0)304        {305            let alt_layout = self306                .layout_of_univariant_biased(fields, repr, kind, NicheBias::End)307                .expect("alt layout should always work");308            let alt_niche = alt_layout309                .largest_niche310                .expect("alt layout should have a niche like the regular one");311            let alt_head_space = alt_niche.offset.bytes();312            let alt_niche_len = alt_niche.value.size(dl).bytes();313            let alt_tail_space = alt_layout.size.bytes() - alt_head_space - alt_niche_len;314315            debug_assert_eq!(layout.size.bytes(), alt_layout.size.bytes());316317            let prefer_alt_layout = alt_head_space > head_space && alt_head_space > tail_space;318319            debug!(320                "sz: {}, default_niche_at: {}+{}, default_tail_space: {}, alt_niche_at/head_space: {}+{}, alt_tail: {}, num_fields: {}, better: {}\n\321                layout: {}\n\322                alt_layout: {}\n",323                layout.size.bytes(),324                head_space,325                niche_len,326                tail_space,327                alt_head_space,328                alt_niche_len,329                alt_tail_space,330                layout.fields.count(),331                prefer_alt_layout,332                self.format_field_niches(layout, fields),333                self.format_field_niches(&alt_layout, fields),334            );335336            if prefer_alt_layout {337                return Ok(alt_layout);338            }339        }340        layout341    }342343    pub fn layout_of_struct_or_enum<'a, FieldIdx, VariantIdx, F>(344        &self,345        repr: &ReprOptions,346        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,347        is_enum: bool,348        niche_optimizations: NicheOptimizations,349        discr_range_of_repr: impl Fn(RangeFrom<i128>, RangeToInclusive<u128>) -> (Integer, bool),350        discriminants: impl Iterator<Item = (VariantIdx, u128)>,351        always_sized: bool,352    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>353    where354        FieldIdx: Idx,355        VariantIdx: Idx,356        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,357    {358        let (present_first, present_second) = {359            let mut present_variants = variants.iter_enumerated().filter_map(|(i, v)| {360                if !repr.inhibit_enum_layout_opt() && absent(v) { None } else { Some(i) }361            });362            (present_variants.next(), present_variants.next())363        };364        let present_first = match present_first {365            Some(present_first) => present_first,366            // Uninhabited because it has no variants, or only absent ones.367            None if is_enum => {368                return Ok(LayoutData::never_type(&self.cx));369            }370            // If it's a struct, still compute a layout so that we can still compute the371            // field offsets.372            None => VariantIdx::new(0),373        };374375        // take the struct path if it is an actual struct376        if !is_enum ||377            // or for optimizing univariant enums378            (present_second.is_none() && !repr.inhibit_enum_layout_opt())379        {380            self.layout_of_struct(381                repr,382                variants,383                present_first,384                is_enum,385                niche_optimizations,386                always_sized,387            )388        } else {389            // At this point, we have handled all unions and390            // structs. (We have also handled univariant enums391            // that allow representation optimization.)392            assert!(is_enum);393            self.layout_of_enum(repr, variants, discr_range_of_repr, discriminants)394        }395    }396397    pub fn layout_of_union<'a, FieldIdx, VariantIdx, F>(398        &self,399        repr: &ReprOptions,400        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,401    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>402    where403        FieldIdx: Idx,404        VariantIdx: Idx,405        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,406    {407        let dl = self.cx.data_layout();408        let mut align = if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align };409        let mut max_repr_align = repr.align;410411        // If all the non-ZST fields have the same repr and union repr optimizations aren't412        // disabled, we can use that common repr for the union as a whole.413        struct AbiMismatch;414        let mut common_non_zst_repr_and_align = if repr.inhibits_union_abi_opt() {415            // Can't optimize416            Err(AbiMismatch)417        } else {418            Ok(None)419        };420421        let mut size = Size::ZERO;422        let only_variant_idx = VariantIdx::new(0);423        let only_variant = &variants[only_variant_idx];424        for field in only_variant {425            if field.is_unsized() {426                return Err(LayoutCalculatorError::UnexpectedUnsized(*field));427            }428429            align = align.max(field.align.abi);430            max_repr_align = max_repr_align.max(field.max_repr_align);431            size = cmp::max(size, field.size);432433            if field.is_zst() {434                // Nothing more to do for ZST fields435                continue;436            }437438            if let Ok(common) = common_non_zst_repr_and_align {439                // Discard valid range information and allow undef440                let field_abi = field.backend_repr.to_union();441442                if let Some((common_abi, common_align)) = common {443                    if common_abi != field_abi {444                        // Different fields have different ABI: disable opt445                        common_non_zst_repr_and_align = Err(AbiMismatch);446                    } else {447                        // Fields with the same non-Aggregate ABI should also448                        // have the same alignment449                        if !matches!(common_abi, BackendRepr::Memory { .. }) {450                            assert_eq!(451                                common_align, field.align.abi,452                                "non-Aggregate field with matching ABI but differing alignment"453                            );454                        }455                    }456                } else {457                    // First non-ZST field: record its ABI and alignment458                    common_non_zst_repr_and_align = Ok(Some((field_abi, field.align.abi)));459                }460            }461        }462463        if let Some(pack) = repr.pack {464            align = align.min(pack);465        }466        // The unadjusted ABI alignment does not include repr(align), but does include repr(pack).467        // See documentation on `LayoutData::unadjusted_abi_align`.468        let unadjusted_abi_align = align;469        if let Some(repr_align) = repr.align {470            align = align.max(repr_align);471        }472        // `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate.473        let align = align;474475        // If all non-ZST fields have the same ABI, we may forward that ABI476        // for the union as a whole, unless otherwise inhibited.477        let backend_repr = match common_non_zst_repr_and_align {478            Err(AbiMismatch) | Ok(None) => BackendRepr::Memory { sized: true },479            Ok(Some((repr, _))) => match repr {480                // Mismatched alignment (e.g. union is #[repr(packed)]): disable opt481                BackendRepr::Scalar(_) | BackendRepr::ScalarPair { .. }482                    if repr.scalar_platform_align(dl).unwrap() != align =>483                {484                    BackendRepr::Memory { sized: true }485                }486                // Vectors require at least element alignment, else disable the opt487                BackendRepr::SimdVector { element, count: _ }488                    if element.default_align(dl).abi > align =>489                {490                    BackendRepr::Memory { sized: true }491                }492                // the alignment tests passed and we can use this493                BackendRepr::Scalar(..)494                | BackendRepr::ScalarPair { .. }495                | BackendRepr::SimdVector { .. }496                | BackendRepr::SimdScalableVector { .. }497                | BackendRepr::Memory { .. } => repr,498            },499        };500501        let Some(union_field_count) = NonZero::new(only_variant.len()) else {502            return Err(LayoutCalculatorError::EmptyUnion);503        };504505        // We also need to check for a `repr(transparent)` wrapper around `repr(C)`506        let repr_c =507            repr.c() || (repr.transparent() && only_variant.iter().any(|field| field.repr_c));508509        let combined_seed = only_variant510            .iter()511            .map(|v| v.randomization_seed)512            .fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));513514        Ok(LayoutData {515            variants: Variants::Single { index: only_variant_idx },516            fields: FieldsShape::Union(union_field_count),517            backend_repr,518            largest_niche: None,519            uninhabited: false,520            align: AbiAlign::new(align),521            size: size.align_to(align),522            max_repr_align,523            unadjusted_abi_align,524            repr_c,525            randomization_seed: combined_seed,526        })527    }528529    /// Calculate the layout for a struct, or a single-variant enum.530    ///531    /// They are the same thing, if you think about it532    /// (Typechecking will reject discriminant-sizing attrs.)533    fn layout_of_struct<'a, FieldIdx, VariantIdx, F>(534        &self,535        repr: &ReprOptions,536        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,537        variant_idx: VariantIdx,538        is_enum: bool,539        niche_optimizations: NicheOptimizations,540        always_sized: bool,541    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>542    where543        FieldIdx: Idx,544        VariantIdx: Idx,545        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,546    {547        let dl = self.cx.data_layout();548        let v = variant_idx;549        let kind = if is_enum || variants[v].is_empty() || always_sized {550            StructKind::AlwaysSized551        } else {552            StructKind::MaybeUnsized553        };554555        let mut st = self.layout_of_univariant(&variants[v], repr, kind)?;556        st.variants = Variants::Single { index: v };557558        if niche_optimizations == NicheOptimizations::Disabled {559            let hide_niches = |scalar: &mut _| match scalar {560                Scalar::Initialized { value, valid_range } => {561                    *valid_range = WrappingRange::full(value.size(dl))562                }563                // Already doesn't have any niches564                Scalar::Union { .. } => {}565            };566            match &mut st.backend_repr {567                BackendRepr::Scalar(scalar) => hide_niches(scalar),568                BackendRepr::ScalarPair { a, b, b_offset: _ } => {569                    hide_niches(a);570                    hide_niches(b);571                }572                BackendRepr::SimdVector { element, .. }573                | BackendRepr::SimdScalableVector { element, .. } => hide_niches(element),574                BackendRepr::Memory { sized: _ } => {}575            }576            st.largest_niche = None;577            return Ok(st);578        }579580        Ok(st)581    }582583    fn layout_of_enum<'a, FieldIdx, VariantIdx, F>(584        &self,585        repr: &ReprOptions,586        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,587        discr_range_of_repr: impl Fn(RangeFrom<i128>, RangeToInclusive<u128>) -> (Integer, bool),588        discriminants: impl Iterator<Item = (VariantIdx, u128)>,589    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>590    where591        FieldIdx: Idx,592        VariantIdx: Idx,593        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,594    {595        let dl = self.cx.data_layout();596        // bail if the enum has an incoherent repr that cannot be computed597        if repr.packed() {598            return Err(LayoutCalculatorError::ReprConflict);599        }600601        let calculate_niche_filling_layout = || -> Option<LayoutData<FieldIdx, VariantIdx>> {602            struct VariantLayoutInfo {603                align_abi: Align,604            }605606            if repr.inhibit_enum_layout_opt() {607                return None;608            }609610            if variants.len() < 2 {611                return None;612            }613614            let mut align = dl.aggregate_align;615            let mut max_repr_align = repr.align;616            let mut unadjusted_abi_align = align;617            let mut combined_seed = repr.field_shuffle_seed;618619            let mut variants_info = IndexVec::<VariantIdx, _>::with_capacity(variants.len());620            let mut variant_layouts = variants621                .iter()622                .map(|v| {623                    let st = self.layout_of_univariant(v, repr, StructKind::AlwaysSized).ok()?;624625                    variants_info.push(VariantLayoutInfo { align_abi: st.align.abi });626627                    align = align.max(st.align.abi);628                    max_repr_align = max_repr_align.max(st.max_repr_align);629                    unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);630                    combined_seed = combined_seed.wrapping_add(st.randomization_seed);631632                    Some(VariantLayout::from_layout(st))633                })634                .collect::<Option<IndexVec<VariantIdx, _>>>()?;635636            let largest_variant_index = variant_layouts637                .iter_enumerated()638                .max_by_key(|(_i, layout)| layout.size.bytes())639                .map(|(i, _layout)| i)?;640641            let all_indices = variants.indices();642            let needs_disc =643                |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]);644            let niche_variants = RangeInclusive {645                start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(),646                last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(),647            };648649            let count =650                (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1;651652            // Use the largest niche in the largest variant.653            let niche = variant_layouts[largest_variant_index].largest_niche?;654            let (niche_start, niche_scalar) = niche.reserve(dl, count)?;655            let niche_offset = niche.offset;656            let niche_size = niche.value.size(dl);657            let size = variant_layouts[largest_variant_index].size.align_to(align);658659            let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| {660                if i == largest_variant_index {661                    return true;662                }663664                layout.largest_niche = None;665666                if layout.size <= niche_offset {667                    // This variant will fit before the niche.668                    return true;669                }670671                // Determine if it'll fit after the niche.672                let this_align = variants_info[i].align_abi;673                let this_offset = (niche_offset + niche_size).align_to(this_align);674675                if this_offset + layout.size > size {676                    return false;677                }678679                // It'll fit, but we need to make some adjustments.680                for offset in layout.field_offsets.iter_mut() {681                    *offset += this_offset;682                }683684                // It can't be a Scalar or ScalarPair because the offset isn't 0.685                if !layout.is_uninhabited() {686                    layout.backend_repr = BackendRepr::Memory { sized: true };687                }688                layout.size += this_offset;689690                true691            });692693            if !all_variants_fit {694                return None;695            }696697            let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);698699            let others_zst = variant_layouts700                .iter_enumerated()701                .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO);702            let same_size = size == variant_layouts[largest_variant_index].size;703            let same_align = align == variants_info[largest_variant_index].align_abi;704705            let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited());706            let abi = if same_size && same_align && others_zst {707                match variant_layouts[largest_variant_index].backend_repr {708                    // When the total alignment and size match, we can use the709                    // same ABI as the scalar variant with the reserved niche.710                    BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar),711                    BackendRepr::ScalarPair { a: first, b: second, b_offset } => {712                        // Only the niche is guaranteed to be initialised,713                        // so use union layouts for the other primitive.714                        if niche_offset == Size::ZERO {715                            BackendRepr::ScalarPair {716                                a: niche_scalar,717                                b: second.to_union(),718                                b_offset,719                            }720                        } else {721                            BackendRepr::ScalarPair {722                                a: first.to_union(),723                                b: niche_scalar,724                                b_offset,725                            }726                        }727                    }728                    _ => BackendRepr::Memory { sized: true },729                }730            } else {731                BackendRepr::Memory { sized: true }732            };733734            let layout = LayoutData {735                variants: Variants::Multiple {736                    tag: niche_scalar,737                    tag_encoding: TagEncoding::Niche {738                        untagged_variant: largest_variant_index,739                        niche_variants,740                        niche_start,741                    },742                    tag_field: FieldIdx::new(0),743                    variants: variant_layouts,744                },745                fields: FieldsShape::Arbitrary {746                    offsets: [niche_offset].into(),747                    in_memory_order: [FieldIdx::new(0)].into(),748                },749                backend_repr: abi,750                largest_niche,751                uninhabited,752                size,753                align: AbiAlign::new(align),754                max_repr_align,755                unadjusted_abi_align,756                repr_c: repr.c(),757                randomization_seed: combined_seed,758            };759760            Some(layout)761        };762763        let niche_filling_layout = calculate_niche_filling_layout();764765        let discr_type = repr.discr_type();766        let discr_size = Integer::from_attr(dl, discr_type).size();767768        let necessary_discriminants: Vec<u128> = discriminants769            .filter(|&(i, _)| repr.c() || variants[i].iter().all(|f| !f.is_uninhabited()))770            .map(|(_, val)| val)771            .collect();772773        // When picking the integer to use, we respect how the discriminants were written774        // in the original rust code, rather than looking only at the bit pattern.775        let (min_negative, max_positive): (i128, u128) = if discr_type.is_signed() {776            necessary_discriminants.iter().copied().map(|val| discr_size.sign_extend(val)).fold(777                (0_i128, 0_u128),778                |(min, max), val| {779                    if let Ok(val) = u128::try_from(val) {780                        (min, max.max(val))781                    } else {782                        (min.min(val), max)783                    }784                },785            )786        } else {787            // We might have no inhabited variants, so pretend there's at least one.788            (0, necessary_discriminants.iter().copied().max().unwrap_or(0))789        };790        trace!(?min_negative, ?max_positive);791792        let (min_ity, signed) = discr_range_of_repr(793            RangeFrom { start: min_negative },794            RangeToInclusive { last: max_positive },795        ); //Integer::discr_range_of_repr(tcx, ty, &repr, min, max);796797        let mut align = dl.aggregate_align;798        let mut max_repr_align = repr.align;799        let mut unadjusted_abi_align = align;800        let mut combined_seed = repr.field_shuffle_seed;801802        let mut size = Size::ZERO;803804        // We're interested in the smallest alignment, so start large.805        let mut start_align = Align::from_bytes(256).unwrap();806        assert_eq!(Integer::for_align(dl, start_align), None);807808        // repr(C) on an enum tells us to make a (tag, union) layout,809        // so we need to grow the prefix alignment to be at least810        // the alignment of the union. (This value is used both for811        // determining the alignment of the overall enum, and the812        // determining the alignment of the payload after the tag.)813        let mut prefix_align = min_ity.align(dl).abi;814        if repr.c() {815            for fields in variants {816                for field in fields {817                    prefix_align = prefix_align.max(field.align.abi);818                }819            }820        }821822        // Create the set of structs that represent each variant.823        let mut layout_variants = variants824            .iter()825            .map(|field_layouts| {826                let st = self.layout_of_univariant(827                    field_layouts,828                    repr,829                    StructKind::Prefixed(min_ity.size(), prefix_align),830                )?;831                // Find the first field we can't move later832                // to make room for a larger discriminant.833                for field_idx in st.fields.index_by_increasing_offset() {834                    let field = &field_layouts[FieldIdx::new(field_idx)];835                    if !field.is_1zst() {836                        start_align = start_align.min(field.align.abi);837                        break;838                    }839                }840                size = cmp::max(size, st.size);841                align = align.max(st.align.abi);842                max_repr_align = max_repr_align.max(st.max_repr_align);843                unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);844                combined_seed = combined_seed.wrapping_add(st.randomization_seed);845                Ok(VariantLayout::from_layout(st))846            })847            .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;848849        // Align the maximum variant size to the largest alignment.850        size = size.align_to(align);851852        // FIXME(oli-obk): deduplicate and harden these checks853        if size.bytes() >= dl.obj_size_bound() {854            return Err(LayoutCalculatorError::SizeOverflow);855        }856857        let typeck_ity = Integer::from_attr(dl, repr.discr_type());858        if typeck_ity < min_ity {859            // It is a bug if Layout decided on a greater discriminant size than typeck for860            // some reason at this point (based on values discriminant can take on). Mostly861            // because this discriminant will be loaded, and then stored into variable of862            // type calculated by typeck. Consider such case (a bug): typeck decided on863            // byte-sized discriminant, but layout thinks we need a 16-bit to store all864            // discriminant values. That would be a bug, because then, in codegen, in order865            // to store this 16-bit discriminant into 8-bit sized temporary some of the866            // space necessary to represent would have to be discarded (or layout is wrong867            // on thinking it needs 16 bits)868            panic!(869                "layout decided on a larger discriminant type ({min_ity:?}) than typeck ({typeck_ity:?})"870            );871            // However, it is fine to make discr type however large (as an optimisation)872            // after this point – we’ll just truncate the value we load in codegen.873        }874875        // Check to see if we should use a different type for the876        // discriminant. We can safely use a type with the same size877        // as the alignment of the first field of each variant.878        // We increase the size of the discriminant to avoid LLVM copying879        // padding when it doesn't need to. This normally causes unaligned880        // load/stores and excessive memcpy/memset operations. By using a881        // bigger integer size, LLVM can be sure about its contents and882        // won't be so conservative.883884        // Use the initial field alignment885        let mut ity = if repr.c() || repr.int.is_some() {886            min_ity887        } else {888            Integer::for_align(dl, start_align).unwrap_or(min_ity)889        };890891        // If the alignment is not larger than the chosen discriminant size,892        // don't use the alignment as the final size.893        if ity <= min_ity {894            ity = min_ity;895        } else {896            // Patch up the variants' first few fields.897            let old_ity_size = min_ity.size();898            let new_ity_size = ity.size();899            for variant in &mut layout_variants {900                for i in &mut variant.field_offsets {901                    if *i <= old_ity_size {902                        assert_eq!(*i, old_ity_size);903                        *i = new_ity_size;904                    }905                }906                // We might be making the struct larger.907                if variant.size <= old_ity_size {908                    variant.size = new_ity_size;909                }910            }911        }912913        let tag_valid_range = {914            let tag_size = ity.size();915            let tags = necessary_discriminants.into_iter().map(|d| tag_size.truncate(d));916            WrappingRange::smallest_range_containing(tags, tag_size)917                // We might have no inhabited variants, so pretend there's at least one.918                .unwrap_or(WrappingRange { start: 0, end: 0 })919        };920        let tag = Scalar::Initialized {921            value: Primitive::Int(ity, signed),922            valid_range: tag_valid_range,923        };924        let mut abi = BackendRepr::Memory { sized: true };925926        let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited());927        if tag.size(dl) == size {928            // Make sure we only use scalar layout when the enum is entirely its929            // own tag (i.e. it has no padding nor any non-ZST variant fields).930            abi = BackendRepr::Scalar(tag);931        } else {932            // Try to use a ScalarPair for all tagged enums.933            // That's possible only if we can find a common primitive type for all variants.934            let mut common_prim = None;935            let mut common_prim_initialized_in_all_variants = true;936            for (field_layouts, layout_variant) in iter::zip(variants, &layout_variants) {937                // We skip *all* ZST here and later check if we are good in terms of alignment.938                // This lets us handle some cases involving aligned ZST.939                let mut fields = iter::zip(field_layouts, &layout_variant.field_offsets)940                    .filter(|p| !p.0.is_zst());941                let (field, offset) = match (fields.next(), fields.next()) {942                    (None, None) => {943                        common_prim_initialized_in_all_variants = false;944                        continue;945                    }946                    (Some(pair), None) => pair,947                    _ => {948                        common_prim = None;949                        break;950                    }951                };952                let prim = match field.backend_repr {953                    BackendRepr::Scalar(scalar) => {954                        common_prim_initialized_in_all_variants &=955                            matches!(scalar, Scalar::Initialized { .. });956                        scalar.primitive()957                    }958                    _ => {959                        common_prim = None;960                        break;961                    }962                };963                if let Some((old_prim, common_offset)) = common_prim {964                    // All variants must be at the same offset965                    if offset != common_offset {966                        common_prim = None;967                        break;968                    }969                    // This is pretty conservative. We could go fancier970                    // by realising that (u8, u8) could just cohabit with971                    // u16 or even u32.972                    let new_prim = match (old_prim, prim) {973                        // Allow all identical primitives.974                        (x, y) if x == y => x,975                        // Allow integers of the same size with differing signedness.976                        // We arbitrarily choose the signedness of the first variant.977                        (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p,978                        // Allow integers mixed with pointers of the same layout.979                        // We must represent this using a pointer, to avoid980                        // roundtripping pointers through ptrtoint/inttoptr.981                        (p @ Primitive::Pointer(_), i @ Primitive::Int(..))982                        | (i @ Primitive::Int(..), p @ Primitive::Pointer(_))983                            if p.size(dl) == i.size(dl)984                                && p.default_align(dl) == i.default_align(dl) =>985                        {986                            p987                        }988                        _ => {989                            common_prim = None;990                            break;991                        }992                    };993                    // We may be updating the primitive here, for example from int->ptr.994                    common_prim = Some((new_prim, common_offset));995                } else {996                    common_prim = Some((prim, offset));997                }998            }999            if let Some((prim, offset)) = common_prim {1000                let prim_scalar = if common_prim_initialized_in_all_variants {1001                    let size = prim.size(dl);1002                    assert!(size.bits() <= 128);1003                    Scalar::Initialized { value: prim, valid_range: WrappingRange::full(size) }1004                } else {1005                    // Common prim might be uninit.1006                    Scalar::Union { value: prim }1007                };1008                let pair =1009                    LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, tag, prim_scalar);1010                let pair_offsets = match pair.fields {1011                    FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {1012                        assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]);1013                        offsets1014                    }1015                    _ => panic!("encountered a non-arbitrary layout during enum layout"),1016                };1017                if pair_offsets[FieldIdx::new(0)] == Size::ZERO1018                    && pair_offsets[FieldIdx::new(1)] == *offset1019                    && align == pair.align.abi1020                    && size == pair.size1021                {1022                    // We can use `ScalarPair` only when it matches our1023                    // already computed layout (including `#[repr(C)]`).1024                    abi = pair.backend_repr;1025                }1026            }1027        }10281029        // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the1030        // variants to ensure they are consistent. This is because a downcast is1031        // semantically a NOP, and thus should not affect layout.1032        if matches!(abi, BackendRepr::Scalar(..) | BackendRepr::ScalarPair { .. }) {1033            for variant in &mut layout_variants {1034                // We only do this for variants with fields; the others are not accessed anyway.1035                // Also do not overwrite any already existing "clever" ABIs.1036                if matches!(variant.backend_repr, BackendRepr::Memory { .. } if variant.has_fields())1037                {1038                    variant.backend_repr = abi;1039                    // Also need to bump up the size, so that the entire value fits in here.1040                    variant.size = cmp::max(variant.size, size);1041                }1042            }1043        }10441045        let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);10461047        let tagged_layout = LayoutData {1048            variants: Variants::Multiple {1049                tag,1050                tag_encoding: TagEncoding::Direct,1051                tag_field: FieldIdx::new(0),1052                variants: layout_variants,1053            },1054            fields: FieldsShape::Arbitrary {1055                offsets: [Size::ZERO].into(),1056                in_memory_order: [FieldIdx::new(0)].into(),1057            },1058            largest_niche,1059            uninhabited,1060            backend_repr: abi,1061            align: AbiAlign::new(align),1062            size,1063            max_repr_align,1064            unadjusted_abi_align,1065            repr_c: repr.c(),1066            randomization_seed: combined_seed,1067        };10681069        let best_layout = match (tagged_layout, niche_filling_layout) {1070            (tl, Some(nl)) => {1071                // Pick the smaller layout; otherwise,1072                // pick the layout with the larger niche; otherwise,1073                // pick tagged as it has simpler codegen.1074                use cmp::Ordering::*;1075                let niche_size = |l: &LayoutData<FieldIdx, VariantIdx>| {1076                    l.largest_niche.map_or(0, |n| n.available(dl))1077                };1078                match (tl.size.cmp(&nl.size), niche_size(&tl).cmp(&niche_size(&nl))) {1079                    (Greater, _) => nl,1080                    (Equal, Less) => nl,1081                    _ => tl,1082                }1083            }1084            (tl, None) => tl,1085        };10861087        Ok(best_layout)1088    }10891090    fn layout_of_univariant_biased<'a, FieldIdx, VariantIdx, F>(1091        &self,1092        fields: &IndexSlice<FieldIdx, F>,1093        repr: &ReprOptions,1094        kind: StructKind,1095        niche_bias: NicheBias,1096    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>1097    where1098        FieldIdx: Idx,1099        VariantIdx: Idx,1100        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,1101    {1102        let dl = self.cx.data_layout();1103        let pack = repr.pack;1104        let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };1105        let mut max_repr_align = repr.align;1106        let mut in_memory_order: IndexVec<u32, FieldIdx> = fields.indices().collect();1107        let optimize_field_order = !repr.inhibit_struct_field_reordering();1108        let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };1109        let optimizing = &mut in_memory_order.raw[..end];1110        let fields_excluding_tail = &fields.raw[..end];1111        // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts.1112        let field_seed = fields_excluding_tail1113            .iter()1114            .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed));11151116        if optimize_field_order && fields.len() > 1 {1117            // If `-Z randomize-layout` was enabled for the type definition we can shuffle1118            // the field ordering to try and catch some code making assumptions about layouts1119            // we don't guarantee.1120            // In the future, we might do more than shuffle field order (e.g. introduce extra padding),1121            // but never for `repr(Rust)` structs with only zero-sized fields, single-variant1122            // `repr(Rust)` enums with only zero-sized fields, or zero-variant `repr(Rust)` enums,1123            // which must remain zero-sized as per T-lang decisions in1124            // https://github.com/rust-lang/reference/pull/2262 and https://github.com/rust-lang/reference/pull/22931125            if repr.can_randomize_type_layout() && cfg!(feature = "randomize") {1126                #[cfg(feature = "randomize")]1127                {1128                    use rand::SeedableRng;1129                    use rand::seq::SliceRandom;1130                    // `ReprOptions.field_shuffle_seed` is a deterministic seed we can use to randomize field1131                    // ordering.1132                    let mut rng = rand_xoshiro::Xoshiro128StarStar::seed_from_u64(1133                        field_seed.wrapping_add(repr.field_shuffle_seed).as_u64(),1134                    );11351136                    // Shuffle the ordering of the fields.1137                    optimizing.shuffle(&mut rng);1138                }1139                // Otherwise we just leave things alone and actually optimize the type's fields1140            } else {1141                // To allow unsizing `&Foo<Type>` -> `&Foo<dyn Trait>`, the layout of the struct must1142                // not depend on the layout of the tail.1143                let max_field_align =1144                    fields_excluding_tail.iter().map(|f| f.align.bytes()).max().unwrap_or(1);1145                let largest_niche_size = fields_excluding_tail1146                    .iter()1147                    .filter_map(|f| f.largest_niche)1148                    .map(|n| n.available(dl))1149                    .max()1150                    .unwrap_or(0);11511152                // Calculates a sort key to group fields by their alignment or possibly some1153                // size-derived pseudo-alignment.1154                let alignment_group_key = |layout: &F| {1155                    // The two branches here return values that cannot be meaningfully compared with1156                    // each other. However, we know that consistently for all executions of1157                    // `alignment_group_key`, one or the other branch will be taken, so this is okay.1158                    if let Some(pack) = pack {1159                        // Return the packed alignment in bytes.1160                        layout.align.abi.min(pack).bytes()1161                    } else {1162                        // Returns `log2(effective-align)`. The calculation assumes that size is an1163                        // integer multiple of align, except for ZSTs.1164                        let align = layout.align.bytes();1165                        let size = layout.size.bytes();1166                        let niche_size = layout.largest_niche.map(|n| n.available(dl)).unwrap_or(0);1167                        // Group [u8; 4] with align-4 or [u8; 6] with align-2 fields.1168                        let size_as_align = align.max(size).trailing_zeros();1169                        let size_as_align = if largest_niche_size > 0 {1170                            match niche_bias {1171                                // Given `A(u8, [u8; 16])` and `B(bool, [u8; 16])` we want to bump the1172                                // array to the front in the first case (for aligned loads) but keep1173                                // the bool in front in the second case for its niches.1174                                NicheBias::Start => {1175                                    max_field_align.trailing_zeros().min(size_as_align)1176                                }1177                                // When moving niches towards the end of the struct then for1178                                // A((u8, u8, u8, bool), (u8, bool, u8)) we want to keep the first tuple1179                                // in the align-1 group because its bool can be moved closer to the end.1180                                NicheBias::End if niche_size == largest_niche_size => {1181                                    align.trailing_zeros()1182                                }1183                                NicheBias::End => size_as_align,1184                            }1185                        } else {1186                            size_as_align1187                        };1188                        size_as_align as u641189                    }1190                };11911192                match kind {1193                    StructKind::AlwaysSized | StructKind::MaybeUnsized => {1194                        // Currently `LayoutData` only exposes a single niche so sorting is usually1195                        // sufficient to get one niche into the preferred position. If it ever1196                        // supported multiple niches then a more advanced pick-and-pack approach could1197                        // provide better results. But even for the single-niche cache it's not1198                        // optimal. E.g. for A(u32, (bool, u8), u16) it would be possible to move the1199                        // bool to the front but it would require packing the tuple together with the1200                        // u16 to build a 4-byte group so that the u32 can be placed after it without1201                        // padding. This kind of packing can't be achieved by sorting.1202                        optimizing.sort_by_key(|&x| {1203                            let f = &fields[x];1204                            let field_size = f.size.bytes();1205                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));1206                            let niche_size_key = match niche_bias {1207                                // large niche first1208                                NicheBias::Start => !niche_size,1209                                // large niche last1210                                NicheBias::End => niche_size,1211                            };1212                            let inner_niche_offset_key = match niche_bias {1213                                NicheBias::Start => f.largest_niche.map_or(0, |n| n.offset.bytes()),1214                                NicheBias::End => f.largest_niche.map_or(0, |n| {1215                                    !(field_size - n.value.size(dl).bytes() - n.offset.bytes())1216                                }),1217                            };12181219                            (1220                                // Then place largest alignments first.1221                                cmp::Reverse(alignment_group_key(f)),1222                                // Then prioritize niche placement within alignment group according to1223                                // `niche_bias_start`.1224                                niche_size_key,1225                                // Then among fields with equally-sized niches prefer the ones1226                                // closer to the start/end of the field.1227                                inner_niche_offset_key,1228                            )1229                        });1230                    }12311232                    StructKind::Prefixed(..) => {1233                        // Sort in ascending alignment so that the layout stays optimal1234                        // regardless of the prefix.1235                        // And put the largest niche in an alignment group at the end1236                        // so it can be used as discriminant in jagged enums1237                        optimizing.sort_by_key(|&x| {1238                            let f = &fields[x];1239                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));1240                            (alignment_group_key(f), niche_size)1241                        });1242                    }1243                }12441245                // FIXME(Kixiron): We can always shuffle fields within a given alignment class1246                //                 regardless of the status of `-Z randomize-layout`1247            }1248        }1249        // in_memory_order holds field indices by increasing memory offset.1250        // That is, if field 5 has offset 0, the first element of in_memory_order is 5.1251        // We now write field offsets to the corresponding offset slot;1252        // field 5 with offset 0 puts 0 in offsets[5].1253        let mut unsized_field = None::<&F>;1254        let mut offsets = IndexVec::from_elem(Size::ZERO, fields);1255        let mut offset = Size::ZERO;1256        let mut largest_niche = None;1257        let mut largest_niche_available = 0;1258        if let StructKind::Prefixed(prefix_size, prefix_align) = kind {1259            let prefix_align =1260                if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };1261            align = align.max(prefix_align);1262            offset = prefix_size.align_to(prefix_align);1263        }1264        for &i in &in_memory_order {1265            let field = &fields[i];1266            if let Some(unsized_field) = unsized_field {1267                return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field));1268            }12691270            if field.is_unsized() {1271                if let StructKind::MaybeUnsized = kind {1272                    unsized_field = Some(field);1273                } else {1274                    return Err(LayoutCalculatorError::UnexpectedUnsized(*field));1275                }1276            }12771278            // Invariant: offset < dl.obj_size_bound() <= 1<<611279            let field_align = if let Some(pack) = pack {1280                field.align.min(AbiAlign::new(pack))1281            } else {1282                field.align1283            };1284            offset = offset.align_to(field_align.abi);1285            align = align.max(field_align.abi);1286            max_repr_align = max_repr_align.max(field.max_repr_align);12871288            debug!("univariant offset: {:?} field: {:#?}", offset, field);1289            offsets[i] = offset;12901291            if let Some(mut niche) = field.largest_niche {1292                let available = niche.available(dl);1293                // Pick up larger niches.1294                let prefer_new_niche = match niche_bias {1295                    NicheBias::Start => available > largest_niche_available,1296                    // if there are several niches of the same size then pick the last one1297                    NicheBias::End => available >= largest_niche_available,1298                };1299                if prefer_new_niche {1300                    largest_niche_available = available;1301                    niche.offset += offset;1302                    largest_niche = Some(niche);1303                }1304            }13051306            offset =1307                offset.checked_add(field.size, dl).ok_or(LayoutCalculatorError::SizeOverflow)?;1308        }13091310        // The unadjusted ABI alignment does not include repr(align), but does include repr(pack).1311        // See documentation on `LayoutData::unadjusted_abi_align`.1312        let unadjusted_abi_align = align;1313        if let Some(repr_align) = repr.align {1314            align = align.max(repr_align);1315        }1316        // `align` must not be modified after this point, or `unadjusted_abi_align` could be inaccurate.1317        let align = align;13181319        debug!("univariant min_size: {:?}", offset);1320        let min_size = offset;1321        let size = min_size.align_to(align);1322        // FIXME(oli-obk): deduplicate and harden these checks1323        if size.bytes() >= dl.obj_size_bound() {1324            return Err(LayoutCalculatorError::SizeOverflow);1325        }1326        let mut layout_of_single_non_zst_field = None;1327        let sized = unsized_field.is_none();1328        let mut abi = BackendRepr::Memory { sized };13291330        let optimize_abi = !repr.inhibit_newtype_abi_optimization();13311332        // Try to make this a Scalar/ScalarPair.1333        if sized && size.bytes() > 0 {1334            // We skip *all* ZST here and later check if we are good in terms of alignment.1335            // This lets us handle some cases involving aligned ZST.1336            let mut non_zst_fields = fields.iter_enumerated().filter(|&(_, f)| !f.is_zst());13371338            match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {1339                // We have exactly one non-ZST field.1340                (Some((i, field)), None, None) => {1341                    layout_of_single_non_zst_field = Some(field);13421343                    // Field fills the struct and it has a scalar or scalar pair ABI.1344                    if offsets[i].bytes() == 0 && align == field.align.abi && size == field.size {1345                        match field.backend_repr {1346                            // For plain scalars, or vectors of them, we can't unpack1347                            // newtypes for `#[repr(C)]`, as that affects C ABIs.1348                            BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. }1349                                if optimize_abi =>1350                            {1351                                abi = field.backend_repr;1352                            }1353                            // But scalar pairs are Rust-specific and get1354                            // treated as aggregates by C ABIs anyway.1355                            BackendRepr::ScalarPair { .. } => {1356                                abi = field.backend_repr;1357                            }1358                            _ => {}1359                        }1360                    }1361                }13621363                // Two non-ZST fields, and they're both scalars.1364                (Some((i, a)), Some((j, b)), None) => {1365                    match (a.backend_repr, b.backend_repr) {1366                        (BackendRepr::Scalar(a), BackendRepr::Scalar(b)) => {1367                            // Order by the memory placement, not source order.1368                            let ((i, a), (j, b)) = if offsets[i] < offsets[j] {1369                                ((i, a), (j, b))1370                            } else {1371                                ((j, b), (i, a))1372                            };1373                            let pair =1374                                LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, a, b);1375                            let pair_offsets = match pair.fields {1376                                FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {1377                                    assert_eq!(1378                                        in_memory_order.raw,1379                                        [FieldIdx::new(0), FieldIdx::new(1)]1380                                    );1381                                    offsets1382                                }1383                                FieldsShape::Primitive1384                                | FieldsShape::Array { .. }1385                                | FieldsShape::Union(..) => {1386                                    panic!("encountered a non-arbitrary layout during enum layout")1387                                }1388                            };1389                            if offsets[i] == pair_offsets[FieldIdx::new(0)]1390                                && offsets[j] == pair_offsets[FieldIdx::new(1)]1391                                && align == pair.align.abi1392                                && size == pair.size1393                            {1394                                // We can use `ScalarPair` only when it matches our1395                                // already computed layout (including `#[repr(C)]`).1396                                abi = pair.backend_repr;1397                            }1398                        }1399                        _ => {}1400                    }1401                }14021403                _ => {}1404            }1405        }1406        let uninhabited = fields.iter().any(|f| f.is_uninhabited());14071408        let unadjusted_abi_align = if repr.transparent() {1409            match layout_of_single_non_zst_field {1410                Some(l) => l.unadjusted_abi_align,1411                None => {1412                    // `repr(transparent)` with all ZST fields.1413                    align1414                }1415            }1416        } else {1417            unadjusted_abi_align1418        };14191420        // We also need to check for a `repr(transparent)` wrapper around `repr(C)`1421        let repr_c = repr.c() || (repr.transparent() && fields.iter().any(|field| field.repr_c));14221423        let seed = field_seed.wrapping_add(repr.field_shuffle_seed);14241425        Ok(LayoutData {1426            variants: Variants::Single { index: VariantIdx::new(0) },1427            fields: FieldsShape::Arbitrary { offsets, in_memory_order },1428            backend_repr: abi,1429            largest_niche,1430            uninhabited,1431            align: AbiAlign::new(align),1432            size,1433            max_repr_align,1434            unadjusted_abi_align,1435            repr_c,1436            randomization_seed: seed,1437        })1438    }14391440    fn format_field_niches<'a, FieldIdx, VariantIdx, F>(1441        &self,1442        layout: &LayoutData<FieldIdx, VariantIdx>,1443        fields: &IndexSlice<FieldIdx, F>,1444    ) -> String1445    where1446        FieldIdx: Idx,1447        VariantIdx: Idx,1448        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,1449    {1450        let dl = self.cx.data_layout();1451        let mut s = String::new();1452        for i in layout.fields.index_by_increasing_offset() {1453            let offset = layout.fields.offset(i);1454            let f = &fields[FieldIdx::new(i)];1455            write!(s, "[o{}a{}s{}", offset.bytes(), f.align.bytes(), f.size.bytes()).unwrap();1456            if let Some(n) = f.largest_niche {1457                write!(1458                    s,1459                    " n{}b{}s{}",1460                    n.offset.bytes(),1461                    n.available(dl).ilog2(),1462                    n.value.size(dl).bytes()1463                )1464                .unwrap();1465            }1466            write!(s, "] ").unwrap();1467        }1468        s1469    }1470}14711472enum SimdVectorKind {1473    /// `#[rustc_scalable_vector]`1474    Scalable(NumScalableVectors),1475    /// `#[repr(simd, packed)]`1476    PackedFixed,1477    /// `#[repr(simd)]`1478    Fixed,1479}14801481fn vector_type_layout<FieldIdx, VariantIdx, F>(1482    kind: SimdVectorKind,1483    dl: &TargetDataLayout,1484    element: F,1485    count: u64,1486) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>1487where1488    FieldIdx: Idx,1489    VariantIdx: Idx,1490    F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,1491{1492    let elt = element.as_ref();1493    let count = BackendLaneCount::new(count)?;14941495    let BackendRepr::Scalar(element) = elt.backend_repr else {1496        return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));1497    };14981499    // Compute the size and alignment of the vector1500    let size =1501        elt.size.checked_mul(count.as_u64(), dl).ok_or(LayoutCalculatorError::SizeOverflow)?;1502    let (repr, size, align) = match kind {1503        SimdVectorKind::Scalable(number_of_vectors) => (1504            BackendRepr::SimdScalableVector { element, count, number_of_vectors },1505            size.checked_mul(number_of_vectors.0 as u64, dl)1506                .ok_or(LayoutCalculatorError::SizeOverflow)?,1507            dl.rust_vector_align(size),1508        ),1509        // Non-power-of-two vectors have padding up to the next power-of-two.1510        // If we're a packed repr, remove the padding while keeping the alignment as close1511        // to a vector as possible.1512        SimdVectorKind::PackedFixed if !count.is_power_of_two() => {1513            (BackendRepr::Memory { sized: true }, size, Align::max_aligned_factor(size))1514        }1515        SimdVectorKind::PackedFixed | SimdVectorKind::Fixed => {1516            (BackendRepr::SimdVector { element, count }, size, dl.rust_vector_align(size))1517        }1518    };1519    let size = size.align_to(align);15201521    Ok(LayoutData {1522        variants: Variants::Single { index: VariantIdx::new(0) },1523        fields: FieldsShape::Arbitrary {1524            offsets: [Size::ZERO].into(),1525            in_memory_order: [FieldIdx::new(0)].into(),1526        },1527        backend_repr: repr,1528        largest_niche: elt.largest_niche,1529        uninhabited: false,1530        size,1531        align: AbiAlign::new(align),1532        max_repr_align: None,1533        unadjusted_abi_align: elt.align.abi,1534        repr_c: false,1535        randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count.as_u64())),1536    })1537}