1use std::collections::BTreeSet;2use std::fmt::{self, Write};3use std::ops::Deref;4use std::range::RangeInclusive;5use std::{cmp, iter};67use rustc_hashes::Hash64;8use rustc_index::Idx;9use rustc_index::bit_set::BitMatrix;10use tracing::{debug, trace};1112use crate::{13    AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,14    LayoutData, Niche, NonZeroUsize, NumScalableVectors, Primitive, ReprOptions, Scalar, Size,15    StructKind, TagEncoding, TargetDataLayout, VariantLayout, Variants, WrappingRange,16};1718mod coroutine;19mod simple;2021#[cfg(feature = "nightly")]22mod ty;2324#[cfg(feature = "nightly")]25pub use ty::{Layout, TyAbiInterface, TyAndLayout};2627rustc_index::newtype_index! {28    /// The *source-order* index of a field in a variant.29    ///30    /// This is how most code after type checking refers to fields, rather than31    /// using names (as names have hygiene complications and more complex lookup).32    ///33    /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order.34    /// (It is for `repr(C)` `struct`s, however.)35    ///36    /// For example, in the following types,37    /// ```rust38    /// # enum Never {}39    /// # #[repr(u16)]40    /// enum Demo1 {41    ///    Variant0 { a: Never, b: i32 } = 100,42    ///    Variant1 { c: u8, d: u64 } = 10,43    /// }44    /// struct Demo2 { e: u8, f: u16, g: u8 }45    /// ```46    /// `b` is `FieldIdx(1)` in `VariantIdx(0)`,47    /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and48    /// `f` is `FieldIdx(1)` in `VariantIdx(0)`.49    #[stable_hash]50    #[encodable]51    #[orderable]52    #[gate_rustc_only]53    pub struct FieldIdx {}54}5556impl FieldIdx {57    /// The second field, at index 1.58    ///59    /// For use alongside [`FieldIdx::ZERO`], particularly with scalar pairs.60    pub const ONE: FieldIdx = FieldIdx::from_u32(1);61}6263rustc_index::newtype_index! {64    /// The *source-order* index of a variant in a type.65    ///66    /// For enums, these are always `0..variant_count`, regardless of any67    /// custom discriminants that may have been defined, and including any68    /// variants that may end up uninhabited due to field types.  (Some of the69    /// variants may not be present in a monomorphized ABI [`Variants`], but70    /// those skipped variants are always counted when determining the *index*.)71    ///72    /// `struct`s, `tuples`, and `unions`s are considered to have a single variant73    /// with variant index zero, aka [`FIRST_VARIANT`].74    #[stable_hash]75    #[encodable]76    #[orderable]77    #[gate_rustc_only]78    pub struct VariantIdx {79        /// Equivalent to `VariantIdx(0)`.80        const FIRST_VARIANT = 0;81    }82}8384// A variant is absent if it's uninhabited and only has ZST fields.85// Present uninhabited variants only require space for their fields,86// but *not* an encoding of the discriminant (e.g., a tag value).87// See issue #49298 for more details on the need to leave space88// for non-ZST uninhabited data (mostly partial initialization).89fn absent<'a, FieldIdx, VariantIdx, F>(fields: &IndexSlice<FieldIdx, F>) -> bool90where91    FieldIdx: Idx,92    VariantIdx: Idx,93    F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,94{95    let uninhabited = fields.iter().any(|f| f.is_uninhabited());96    // We cannot ignore alignment; that might lead us to entirely discard a variant and97    // produce an enum that is less aligned than it should be!98    let is_1zst = fields.iter().all(|f| f.is_1zst());99    uninhabited && is_1zst100}101102/// Determines towards which end of a struct layout optimizations will try to place the best niches.103enum NicheBias {104    Start,105    End,106}107108#[derive(Copy, Clone, Debug, PartialEq, Eq)]109pub enum LayoutCalculatorError<F> {110    /// An unsized type was found in a location where a sized type was expected.111    ///112    /// This is not always a compile error, for example if there is a `[T]: Sized`113    /// bound in a where clause.114    ///115    /// Contains the field that was unexpectedly unsized.116    UnexpectedUnsized(F),117118    /// A type was too large for the target platform.119    SizeOverflow,120121    /// A union had no fields.122    EmptyUnion,123124    /// The fields or variants have irreconcilable reprs125    ReprConflict,126127    /// The length of an SIMD type is zero128    ZeroLengthSimdType,129130    /// The length of an SIMD type exceeds the maximum number of lanes131    OversizedSimdType { max_lanes: u64 },132133    /// An element type of an SIMD type isn't a primitive134    NonPrimitiveSimdType(F),135}136137impl<F> LayoutCalculatorError<F> {138    pub fn without_payload(&self) -> LayoutCalculatorError<()> {139        use LayoutCalculatorError::*;140        match *self {141            UnexpectedUnsized(_) => UnexpectedUnsized(()),142            SizeOverflow => SizeOverflow,143            EmptyUnion => EmptyUnion,144            ReprConflict => ReprConflict,145            ZeroLengthSimdType => ZeroLengthSimdType,146            OversizedSimdType { max_lanes } => OversizedSimdType { max_lanes },147            NonPrimitiveSimdType(_) => NonPrimitiveSimdType(()),148        }149    }150151    /// Format an untranslated diagnostic for this type152    ///153    /// Intended for use by rust-analyzer, as neither it nor `rustc_abi` depend on fluent infra.154    pub fn fallback_fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {155        use LayoutCalculatorError::*;156        f.write_str(match self {157            UnexpectedUnsized(_) => "an unsized type was found where a sized type was expected",158            SizeOverflow => "size overflow",159            EmptyUnion => "type is a union with no fields",160            ReprConflict => "type has an invalid repr",161            ZeroLengthSimdType | OversizedSimdType { .. } | NonPrimitiveSimdType(_) => {162                "invalid simd type definition"163            }164        })165    }166}167168type LayoutCalculatorResult<FieldIdx, VariantIdx, F> =169    Result<LayoutData<FieldIdx, VariantIdx>, LayoutCalculatorError<F>>;170171#[derive(Clone, Copy, Debug)]172pub struct LayoutCalculator<Cx> {173    pub cx: Cx,174}175176impl<Cx: HasDataLayout> LayoutCalculator<Cx> {177    pub fn new(cx: Cx) -> Self {178        Self { cx }179    }180181    pub fn array_like<FieldIdx: Idx, VariantIdx: Idx, F>(182        &self,183        element: &LayoutData<FieldIdx, VariantIdx>,184        count_if_sized: Option<u64>, // None for slices185    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {186        let count = count_if_sized.unwrap_or(0);187        let size =188            element.size.checked_mul(count, &self.cx).ok_or(LayoutCalculatorError::SizeOverflow)?;189190        Ok(LayoutData {191            variants: Variants::Single { index: VariantIdx::new(0) },192            fields: FieldsShape::Array { stride: element.size, count },193            backend_repr: BackendRepr::Memory { sized: count_if_sized.is_some() },194            largest_niche: element.largest_niche.filter(|_| count != 0),195            uninhabited: element.uninhabited && count != 0,196            align: element.align,197            size,198            max_repr_align: None,199            unadjusted_abi_align: element.align.abi,200            randomization_seed: element.randomization_seed.wrapping_add(Hash64::new(count)),201        })202    }203204    pub fn scalable_vector_type<FieldIdx, VariantIdx, F>(205        &self,206        element: F,207        count: u64,208        number_of_vectors: NumScalableVectors,209    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>210    where211        FieldIdx: Idx,212        VariantIdx: Idx,213        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,214    {215        vector_type_layout(216            SimdVectorKind::Scalable(number_of_vectors),217            self.cx.data_layout(),218            element,219            count,220        )221    }222223    pub fn simd_type<FieldIdx, VariantIdx, F>(224        &self,225        element: F,226        count: u64,227        repr_packed: bool,228    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>229    where230        FieldIdx: Idx,231        VariantIdx: Idx,232        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,233    {234        let kind = if repr_packed { SimdVectorKind::PackedFixed } else { SimdVectorKind::Fixed };235        vector_type_layout(kind, self.cx.data_layout(), element, count)236    }237238    /// Compute the layout for a coroutine.239    ///240    /// This uses dedicated code instead of [`Self::layout_of_struct_or_enum`], as coroutine241    /// fields may be shared between multiple variants (see the [`coroutine`] module for details).242    pub fn coroutine<243        'a,244        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,245        VariantIdx: Idx,246        FieldIdx: Idx,247        LocalIdx: Idx,248    >(249        &self,250        local_layouts: &IndexSlice<LocalIdx, F>,251        prefix_layouts: IndexVec<FieldIdx, F>,252        variant_fields: &IndexSlice<VariantIdx, IndexVec<FieldIdx, LocalIdx>>,253        storage_conflicts: &BitMatrix<LocalIdx, LocalIdx>,254        tag_to_layout: impl Fn(Scalar) -> F,255    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {256        coroutine::layout(257            self,258            local_layouts,259            prefix_layouts,260            variant_fields,261            storage_conflicts,262            tag_to_layout,263        )264    }265266    pub fn univariant<267        'a,268        FieldIdx: Idx,269        VariantIdx: Idx,270        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,271    >(272        &self,273        fields: &IndexSlice<FieldIdx, F>,274        repr: &ReprOptions,275        kind: StructKind,276    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {277        let dl = self.cx.data_layout();278        let layout = self.univariant_biased(fields, repr, kind, NicheBias::Start);279        // Enums prefer niches close to the beginning or the end of the variants so that other280        // (smaller) data-carrying variants can be packed into the space after/before the niche.281        // If the default field ordering does not give us a niche at the front then we do a second282        // run and bias niches to the right and then check which one is closer to one of the283        // struct's edges.284        if let Ok(layout) = &layout {285            // Don't try to calculate an end-biased layout for unsizable structs,286            // otherwise we could end up with different layouts for287            // Foo<Type> and Foo<dyn Trait> which would break unsizing.288            if !matches!(kind, StructKind::MaybeUnsized) {289                if let Some(niche) = layout.largest_niche {290                    let head_space = niche.offset.bytes();291                    let niche_len = niche.value.size(dl).bytes();292                    let tail_space = layout.size.bytes() - head_space - niche_len;293294                    // This may end up doing redundant work if the niche is already in the last295                    // field (e.g. a trailing bool) and there is tail padding. But it's non-trivial296                    // to get the unpadded size so we try anyway.297                    if fields.len() > 1 && head_space != 0 && tail_space > 0 {298                        let alt_layout = self299                            .univariant_biased(fields, repr, kind, NicheBias::End)300                            .expect("alt layout should always work");301                        let alt_niche = alt_layout302                            .largest_niche303                            .expect("alt layout should have a niche like the regular one");304                        let alt_head_space = alt_niche.offset.bytes();305                        let alt_niche_len = alt_niche.value.size(dl).bytes();306                        let alt_tail_space =307                            alt_layout.size.bytes() - alt_head_space - alt_niche_len;308309                        debug_assert_eq!(layout.size.bytes(), alt_layout.size.bytes());310311                        let prefer_alt_layout =312                            alt_head_space > head_space && alt_head_space > tail_space;313314                        debug!(315                            "sz: {}, default_niche_at: {}+{}, default_tail_space: {}, alt_niche_at/head_space: {}+{}, alt_tail: {}, num_fields: {}, better: {}\n\316                            layout: {}\n\317                            alt_layout: {}\n",318                            layout.size.bytes(),319                            head_space,320                            niche_len,321                            tail_space,322                            alt_head_space,323                            alt_niche_len,324                            alt_tail_space,325                            layout.fields.count(),326                            prefer_alt_layout,327                            self.format_field_niches(layout, fields),328                            self.format_field_niches(&alt_layout, fields),329                        );330331                        if prefer_alt_layout {332                            return Ok(alt_layout);333                        }334                    }335                }336            }337        }338        layout339    }340341    pub fn layout_of_struct_or_enum<342        'a,343        FieldIdx: Idx,344        VariantIdx: Idx,345        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,346    >(347        &self,348        repr: &ReprOptions,349        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,350        is_enum: bool,351        is_special_no_niche: bool,352        discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),353        discriminants: impl Iterator<Item = (VariantIdx, i128)>,354        always_sized: bool,355    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {356        let (present_first, present_second) = {357            let mut present_variants = variants.iter_enumerated().filter_map(|(i, v)| {358                if !repr.inhibit_enum_layout_opt() && absent(v) { None } else { Some(i) }359            });360            (present_variants.next(), present_variants.next())361        };362        let present_first = match present_first {363            Some(present_first) => present_first,364            // Uninhabited because it has no variants, or only absent ones.365            None if is_enum => {366                return Ok(LayoutData::never_type(&self.cx));367            }368            // If it's a struct, still compute a layout so that we can still compute the369            // field offsets.370            None => VariantIdx::new(0),371        };372373        // take the struct path if it is an actual struct374        if !is_enum ||375            // or for optimizing univariant enums376            (present_second.is_none() && !repr.inhibit_enum_layout_opt())377        {378            self.layout_of_struct(379                repr,380                variants,381                is_enum,382                is_special_no_niche,383                always_sized,384                present_first,385            )386        } else {387            // At this point, we have handled all unions and388            // structs. (We have also handled univariant enums389            // that allow representation optimization.)390            assert!(is_enum);391            self.layout_of_enum(repr, variants, discr_range_of_repr, discriminants)392        }393    }394395    pub fn layout_of_union<396        'a,397        FieldIdx: Idx,398        VariantIdx: Idx,399        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,400    >(401        &self,402        repr: &ReprOptions,403        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,404    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {405        let dl = self.cx.data_layout();406        let mut align = if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align };407        let mut max_repr_align = repr.align;408409        // If all the non-ZST fields have the same repr and union repr optimizations aren't410        // disabled, we can use that common repr for the union as a whole.411        struct AbiMismatch;412        let mut common_non_zst_repr_and_align = if repr.inhibits_union_abi_opt() {413            // Can't optimize414            Err(AbiMismatch)415        } else {416            Ok(None)417        };418419        let mut size = Size::ZERO;420        let only_variant_idx = VariantIdx::new(0);421        let only_variant = &variants[only_variant_idx];422        for field in only_variant {423            if field.is_unsized() {424                return Err(LayoutCalculatorError::UnexpectedUnsized(*field));425            }426427            align = align.max(field.align.abi);428            max_repr_align = max_repr_align.max(field.max_repr_align);429            size = cmp::max(size, field.size);430431            if field.is_zst() {432                // Nothing more to do for ZST fields433                continue;434            }435436            if let Ok(common) = common_non_zst_repr_and_align {437                // Discard valid range information and allow undef438                let field_abi = field.backend_repr.to_union();439440                if let Some((common_abi, common_align)) = common {441                    if common_abi != field_abi {442                        // Different fields have different ABI: disable opt443                        common_non_zst_repr_and_align = Err(AbiMismatch);444                    } else {445                        // Fields with the same non-Aggregate ABI should also446                        // have the same alignment447                        if !matches!(common_abi, BackendRepr::Memory { .. }) {448                            assert_eq!(449                                common_align, field.align.abi,450                                "non-Aggregate field with matching ABI but differing alignment"451                            );452                        }453                    }454                } else {455                    // First non-ZST field: record its ABI and alignment456                    common_non_zst_repr_and_align = Ok(Some((field_abi, field.align.abi)));457                }458            }459        }460461        if let Some(pack) = repr.pack {462            align = align.min(pack);463        }464        // The unadjusted ABI alignment does not include repr(align), but does include repr(pack).465        // See documentation on `LayoutData::unadjusted_abi_align`.466        let unadjusted_abi_align = align;467        if let Some(repr_align) = repr.align {468            align = align.max(repr_align);469        }470        // `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate.471        let align = align;472473        // If all non-ZST fields have the same ABI, we may forward that ABI474        // for the union as a whole, unless otherwise inhibited.475        let backend_repr = match common_non_zst_repr_and_align {476            Err(AbiMismatch) | Ok(None) => BackendRepr::Memory { sized: true },477            Ok(Some((repr, _))) => match repr {478                // Mismatched alignment (e.g. union is #[repr(packed)]): disable opt479                BackendRepr::Scalar(_) | BackendRepr::ScalarPair(_, _)480                    if repr.scalar_align(dl).unwrap() != align =>481                {482                    BackendRepr::Memory { sized: true }483                }484                // Vectors require at least element alignment, else disable the opt485                BackendRepr::SimdVector { element, count: _ } if element.align(dl).abi > align => {486                    BackendRepr::Memory { sized: true }487                }488                // the alignment tests passed and we can use this489                BackendRepr::Scalar(..)490                | BackendRepr::ScalarPair(..)491                | BackendRepr::SimdVector { .. }492                | BackendRepr::SimdScalableVector { .. }493                | BackendRepr::Memory { .. } => repr,494            },495        };496497        let Some(union_field_count) = NonZeroUsize::new(only_variant.len()) else {498            return Err(LayoutCalculatorError::EmptyUnion);499        };500501        let combined_seed = only_variant502            .iter()503            .map(|v| v.randomization_seed)504            .fold(repr.field_shuffle_seed, |acc, seed| acc.wrapping_add(seed));505506        Ok(LayoutData {507            variants: Variants::Single { index: only_variant_idx },508            fields: FieldsShape::Union(union_field_count),509            backend_repr,510            largest_niche: None,511            uninhabited: false,512            align: AbiAlign::new(align),513            size: size.align_to(align),514            max_repr_align,515            unadjusted_abi_align,516            randomization_seed: combined_seed,517        })518    }519520    /// single-variant enums are just structs, if you think about it521    fn layout_of_struct<522        'a,523        FieldIdx: Idx,524        VariantIdx: Idx,525        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,526    >(527        &self,528        repr: &ReprOptions,529        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,530        is_enum: bool,531        is_special_no_niche: bool,532        always_sized: bool,533        present_first: VariantIdx,534    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {535        // Struct, or univariant enum equivalent to a struct.536        // (Typechecking will reject discriminant-sizing attrs.)537538        let dl = self.cx.data_layout();539        let v = present_first;540        let kind = if is_enum || variants[v].is_empty() || always_sized {541            StructKind::AlwaysSized542        } else {543            StructKind::MaybeUnsized544        };545546        let mut st = self.univariant(&variants[v], repr, kind)?;547        st.variants = Variants::Single { index: v };548549        if is_special_no_niche {550            let hide_niches = |scalar: &mut _| match scalar {551                Scalar::Initialized { value, valid_range } => {552                    *valid_range = WrappingRange::full(value.size(dl))553                }554                // Already doesn't have any niches555                Scalar::Union { .. } => {}556            };557            match &mut st.backend_repr {558                BackendRepr::Scalar(scalar) => hide_niches(scalar),559                BackendRepr::ScalarPair(a, b) => {560                    hide_niches(a);561                    hide_niches(b);562                }563                BackendRepr::SimdVector { element, .. }564                | BackendRepr::SimdScalableVector { element, .. } => hide_niches(element),565                BackendRepr::Memory { sized: _ } => {}566            }567            st.largest_niche = None;568            return Ok(st);569        }570571        Ok(st)572    }573574    fn layout_of_enum<575        'a,576        FieldIdx: Idx,577        VariantIdx: Idx,578        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,579    >(580        &self,581        repr: &ReprOptions,582        variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,583        discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),584        discriminants: impl Iterator<Item = (VariantIdx, i128)>,585    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {586        let dl = self.cx.data_layout();587        // bail if the enum has an incoherent repr that cannot be computed588        if repr.packed() {589            return Err(LayoutCalculatorError::ReprConflict);590        }591592        let calculate_niche_filling_layout = || -> Option<LayoutData<FieldIdx, VariantIdx>> {593            struct VariantLayoutInfo {594                align_abi: Align,595            }596597            if repr.inhibit_enum_layout_opt() {598                return None;599            }600601            if variants.len() < 2 {602                return None;603            }604605            let mut align = dl.aggregate_align;606            let mut max_repr_align = repr.align;607            let mut unadjusted_abi_align = align;608            let mut combined_seed = repr.field_shuffle_seed;609610            let mut variants_info = IndexVec::<VariantIdx, _>::with_capacity(variants.len());611            let mut variant_layouts = variants612                .iter()613                .map(|v| {614                    let st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?;615616                    variants_info.push(VariantLayoutInfo { align_abi: st.align.abi });617618                    align = align.max(st.align.abi);619                    max_repr_align = max_repr_align.max(st.max_repr_align);620                    unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);621                    combined_seed = combined_seed.wrapping_add(st.randomization_seed);622623                    Some(VariantLayout::from_layout(st))624                })625                .collect::<Option<IndexVec<VariantIdx, _>>>()?;626627            let largest_variant_index = variant_layouts628                .iter_enumerated()629                .max_by_key(|(_i, layout)| layout.size.bytes())630                .map(|(i, _layout)| i)?;631632            let all_indices = variants.indices();633            let needs_disc =634                |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]);635            let niche_variants = RangeInclusive {636                start: all_indices.clone().find(|v| needs_disc(*v)).unwrap(),637                last: all_indices.rev().find(|v| needs_disc(*v)).unwrap(),638            };639640            let count =641                (niche_variants.last.index() as u128 - niche_variants.start.index() as u128) + 1;642643            // Use the largest niche in the largest variant.644            let niche = variant_layouts[largest_variant_index].largest_niche?;645            let (niche_start, niche_scalar) = niche.reserve(dl, count)?;646            let niche_offset = niche.offset;647            let niche_size = niche.value.size(dl);648            let size = variant_layouts[largest_variant_index].size.align_to(align);649650            let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| {651                if i == largest_variant_index {652                    return true;653                }654655                layout.largest_niche = None;656657                if layout.size <= niche_offset {658                    // This variant will fit before the niche.659                    return true;660                }661662                // Determine if it'll fit after the niche.663                let this_align = variants_info[i].align_abi;664                let this_offset = (niche_offset + niche_size).align_to(this_align);665666                if this_offset + layout.size > size {667                    return false;668                }669670                // It'll fit, but we need to make some adjustments.671                for offset in layout.field_offsets.iter_mut() {672                    *offset += this_offset;673                }674675                // It can't be a Scalar or ScalarPair because the offset isn't 0.676                if !layout.is_uninhabited() {677                    layout.backend_repr = BackendRepr::Memory { sized: true };678                }679                layout.size += this_offset;680681                true682            });683684            if !all_variants_fit {685                return None;686            }687688            let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);689690            let others_zst = variant_layouts691                .iter_enumerated()692                .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO);693            let same_size = size == variant_layouts[largest_variant_index].size;694            let same_align = align == variants_info[largest_variant_index].align_abi;695696            let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited());697            let abi = if same_size && same_align && others_zst {698                match variant_layouts[largest_variant_index].backend_repr {699                    // When the total alignment and size match, we can use the700                    // same ABI as the scalar variant with the reserved niche.701                    BackendRepr::Scalar(_) => BackendRepr::Scalar(niche_scalar),702                    BackendRepr::ScalarPair(first, second) => {703                        // Only the niche is guaranteed to be initialised,704                        // so use union layouts for the other primitive.705                        if niche_offset == Size::ZERO {706                            BackendRepr::ScalarPair(niche_scalar, second.to_union())707                        } else {708                            BackendRepr::ScalarPair(first.to_union(), niche_scalar)709                        }710                    }711                    _ => BackendRepr::Memory { sized: true },712                }713            } else {714                BackendRepr::Memory { sized: true }715            };716717            let layout = LayoutData {718                variants: Variants::Multiple {719                    tag: niche_scalar,720                    tag_encoding: TagEncoding::Niche {721                        untagged_variant: largest_variant_index,722                        niche_variants,723                        niche_start,724                    },725                    tag_field: FieldIdx::new(0),726                    variants: variant_layouts,727                },728                fields: FieldsShape::Arbitrary {729                    offsets: [niche_offset].into(),730                    in_memory_order: [FieldIdx::new(0)].into(),731                },732                backend_repr: abi,733                largest_niche,734                uninhabited,735                size,736                align: AbiAlign::new(align),737                max_repr_align,738                unadjusted_abi_align,739                randomization_seed: combined_seed,740            };741742            Some(layout)743        };744745        let niche_filling_layout = calculate_niche_filling_layout();746747        let discr_type = repr.discr_type();748        let discr_int = Integer::from_attr(dl, discr_type);749        // Because we can only represent one range of valid values, we'll look for the750        // largest range of invalid values and pick everything else as the range of valid751        // values.752753        // First we need to sort the possible discriminant values so that we can look for the largest gap:754        let valid_discriminants: BTreeSet<i128> = discriminants755            .filter(|&(i, _)| repr.c() || variants[i].iter().all(|f| !f.is_uninhabited()))756            .map(|(_, val)| {757                if discr_type.is_signed() {758                    // sign extend the raw representation to be an i128759                    // FIXME: do this at the discriminant iterator creation sites760                    discr_int.size().sign_extend(val as u128)761                } else {762                    val763                }764            })765            .collect();766        trace!(?valid_discriminants);767        let discriminants = valid_discriminants.iter().copied();768        //let next_discriminants = discriminants.clone().cycle().skip(1);769        let next_discriminants =770            discriminants.clone().chain(valid_discriminants.first().copied()).skip(1);771        // Iterate over pairs of each discriminant together with the next one.772        // Since they were sorted, we can now compute the niche sizes and pick the largest.773        let discriminants = discriminants.zip(next_discriminants);774        let largest_niche = discriminants.max_by_key(|&(start, end)| {775            trace!(?start, ?end);776            // If this is a wraparound range, the niche size is `MAX - abs(diff)`, as the diff between777            // the two end points is actually the size of the valid discriminants.778            let dist = if start > end {779                // Overflow can happen for 128 bit discriminants if `end` is negative.780                // But in that case casting to `u128` still gets us the right value,781                // as the distance must be positive if the lhs of the subtraction is larger than the rhs.782                let dist = start.wrapping_sub(end);783                if discr_type.is_signed() {784                    discr_int.signed_max().wrapping_sub(dist) as u128785                } else {786                    discr_int.size().unsigned_int_max() - dist as u128787                }788            } else {789                // Overflow can happen for 128 bit discriminants if `start` is negative.790                // But in that case casting to `u128` still gets us the right value,791                // as the distance must be positive if the lhs of the subtraction is larger than the rhs.792                end.wrapping_sub(start) as u128793            };794            trace!(?dist);795            dist796        });797        trace!(?largest_niche);798799        // `max` is the last valid discriminant before the largest niche800        // `min` is the first valid discriminant after the largest niche801        let (max, min) = largest_niche802            // We might have no inhabited variants, so pretend there's at least one.803            .unwrap_or((0, 0));804        let (min_ity, signed) = discr_range_of_repr(min, max); //Integer::discr_range_of_repr(tcx, ty, &repr, min, max);805806        let mut align = dl.aggregate_align;807        let mut max_repr_align = repr.align;808        let mut unadjusted_abi_align = align;809        let mut combined_seed = repr.field_shuffle_seed;810811        let mut size = Size::ZERO;812813        // We're interested in the smallest alignment, so start large.814        let mut start_align = Align::from_bytes(256).unwrap();815        assert_eq!(Integer::for_align(dl, start_align), None);816817        // repr(C) on an enum tells us to make a (tag, union) layout,818        // so we need to grow the prefix alignment to be at least819        // the alignment of the union. (This value is used both for820        // determining the alignment of the overall enum, and the821        // determining the alignment of the payload after the tag.)822        let mut prefix_align = min_ity.align(dl).abi;823        if repr.c() {824            for fields in variants {825                for field in fields {826                    prefix_align = prefix_align.max(field.align.abi);827                }828            }829        }830831        // Create the set of structs that represent each variant.832        let mut layout_variants = variants833            .iter()834            .map(|field_layouts| {835                let st = self.univariant(836                    field_layouts,837                    repr,838                    StructKind::Prefixed(min_ity.size(), prefix_align),839                )?;840                // Find the first field we can't move later841                // to make room for a larger discriminant.842                for field_idx in st.fields.index_by_increasing_offset() {843                    let field = &field_layouts[FieldIdx::new(field_idx)];844                    if !field.is_1zst() {845                        start_align = start_align.min(field.align.abi);846                        break;847                    }848                }849                size = cmp::max(size, st.size);850                align = align.max(st.align.abi);851                max_repr_align = max_repr_align.max(st.max_repr_align);852                unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align);853                combined_seed = combined_seed.wrapping_add(st.randomization_seed);854                Ok(VariantLayout::from_layout(st))855            })856            .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;857858        // Align the maximum variant size to the largest alignment.859        size = size.align_to(align);860861        // FIXME(oli-obk): deduplicate and harden these checks862        if size.bytes() >= dl.obj_size_bound() {863            return Err(LayoutCalculatorError::SizeOverflow);864        }865866        let typeck_ity = Integer::from_attr(dl, repr.discr_type());867        if typeck_ity < min_ity {868            // It is a bug if Layout decided on a greater discriminant size than typeck for869            // some reason at this point (based on values discriminant can take on). Mostly870            // because this discriminant will be loaded, and then stored into variable of871            // type calculated by typeck. Consider such case (a bug): typeck decided on872            // byte-sized discriminant, but layout thinks we need a 16-bit to store all873            // discriminant values. That would be a bug, because then, in codegen, in order874            // to store this 16-bit discriminant into 8-bit sized temporary some of the875            // space necessary to represent would have to be discarded (or layout is wrong876            // on thinking it needs 16 bits)877            panic!(878                "layout decided on a larger discriminant type ({min_ity:?}) than typeck ({typeck_ity:?})"879            );880            // However, it is fine to make discr type however large (as an optimisation)881            // after this point – we’ll just truncate the value we load in codegen.882        }883884        // Check to see if we should use a different type for the885        // discriminant. We can safely use a type with the same size886        // as the alignment of the first field of each variant.887        // We increase the size of the discriminant to avoid LLVM copying888        // padding when it doesn't need to. This normally causes unaligned889        // load/stores and excessive memcpy/memset operations. By using a890        // bigger integer size, LLVM can be sure about its contents and891        // won't be so conservative.892893        // Use the initial field alignment894        let mut ity = if repr.c() || repr.int.is_some() {895            min_ity896        } else {897            Integer::for_align(dl, start_align).unwrap_or(min_ity)898        };899900        // If the alignment is not larger than the chosen discriminant size,901        // don't use the alignment as the final size.902        if ity <= min_ity {903            ity = min_ity;904        } else {905            // Patch up the variants' first few fields.906            let old_ity_size = min_ity.size();907            let new_ity_size = ity.size();908            for variant in &mut layout_variants {909                for i in &mut variant.field_offsets {910                    if *i <= old_ity_size {911                        assert_eq!(*i, old_ity_size);912                        *i = new_ity_size;913                    }914                }915                // We might be making the struct larger.916                if variant.size <= old_ity_size {917                    variant.size = new_ity_size;918                }919            }920        }921922        let tag_mask = ity.size().unsigned_int_max();923        let tag = Scalar::Initialized {924            value: Primitive::Int(ity, signed),925            valid_range: WrappingRange {926                start: (min as u128 & tag_mask),927                end: (max as u128 & tag_mask),928            },929        };930        let mut abi = BackendRepr::Memory { sized: true };931932        let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited());933        if tag.size(dl) == size {934            // Make sure we only use scalar layout when the enum is entirely its935            // own tag (i.e. it has no padding nor any non-ZST variant fields).936            abi = BackendRepr::Scalar(tag);937        } else {938            // Try to use a ScalarPair for all tagged enums.939            // That's possible only if we can find a common primitive type for all variants.940            let mut common_prim = None;941            let mut common_prim_initialized_in_all_variants = true;942            for (field_layouts, layout_variant) in iter::zip(variants, &layout_variants) {943                // We skip *all* ZST here and later check if we are good in terms of alignment.944                // This lets us handle some cases involving aligned ZST.945                let mut fields = iter::zip(field_layouts, &layout_variant.field_offsets)946                    .filter(|p| !p.0.is_zst());947                let (field, offset) = match (fields.next(), fields.next()) {948                    (None, None) => {949                        common_prim_initialized_in_all_variants = false;950                        continue;951                    }952                    (Some(pair), None) => pair,953                    _ => {954                        common_prim = None;955                        break;956                    }957                };958                let prim = match field.backend_repr {959                    BackendRepr::Scalar(scalar) => {960                        common_prim_initialized_in_all_variants &=961                            matches!(scalar, Scalar::Initialized { .. });962                        scalar.primitive()963                    }964                    _ => {965                        common_prim = None;966                        break;967                    }968                };969                if let Some((old_prim, common_offset)) = common_prim {970                    // All variants must be at the same offset971                    if offset != common_offset {972                        common_prim = None;973                        break;974                    }975                    // This is pretty conservative. We could go fancier976                    // by realising that (u8, u8) could just cohabit with977                    // u16 or even u32.978                    let new_prim = match (old_prim, prim) {979                        // Allow all identical primitives.980                        (x, y) if x == y => x,981                        // Allow integers of the same size with differing signedness.982                        // We arbitrarily choose the signedness of the first variant.983                        (p @ Primitive::Int(x, _), Primitive::Int(y, _)) if x == y => p,984                        // Allow integers mixed with pointers of the same layout.985                        // We must represent this using a pointer, to avoid986                        // roundtripping pointers through ptrtoint/inttoptr.987                        (p @ Primitive::Pointer(_), i @ Primitive::Int(..))988                        | (i @ Primitive::Int(..), p @ Primitive::Pointer(_))989                            if p.size(dl) == i.size(dl) && p.align(dl) == i.align(dl) =>990                        {991                            p992                        }993                        _ => {994                            common_prim = None;995                            break;996                        }997                    };998                    // We may be updating the primitive here, for example from int->ptr.999                    common_prim = Some((new_prim, common_offset));1000                } else {1001                    common_prim = Some((prim, offset));1002                }1003            }1004            if let Some((prim, offset)) = common_prim {1005                let prim_scalar = if common_prim_initialized_in_all_variants {1006                    let size = prim.size(dl);1007                    assert!(size.bits() <= 128);1008                    Scalar::Initialized { value: prim, valid_range: WrappingRange::full(size) }1009                } else {1010                    // Common prim might be uninit.1011                    Scalar::Union { value: prim }1012                };1013                let pair =1014                    LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, tag, prim_scalar);1015                let pair_offsets = match pair.fields {1016                    FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {1017                        assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]);1018                        offsets1019                    }1020                    _ => panic!("encountered a non-arbitrary layout during enum layout"),1021                };1022                if pair_offsets[FieldIdx::new(0)] == Size::ZERO1023                    && pair_offsets[FieldIdx::new(1)] == *offset1024                    && align == pair.align.abi1025                    && size == pair.size1026                {1027                    // We can use `ScalarPair` only when it matches our1028                    // already computed layout (including `#[repr(C)]`).1029                    abi = pair.backend_repr;1030                }1031            }1032        }10331034        // If we pick a "clever" (by-value) ABI, we might have to adjust the ABI of the1035        // variants to ensure they are consistent. This is because a downcast is1036        // semantically a NOP, and thus should not affect layout.1037        if matches!(abi, BackendRepr::Scalar(..) | BackendRepr::ScalarPair(..)) {1038            for variant in &mut layout_variants {1039                // We only do this for variants with fields; the others are not accessed anyway.1040                // Also do not overwrite any already existing "clever" ABIs.1041                if matches!(variant.backend_repr, BackendRepr::Memory { .. } if variant.has_fields())1042                {1043                    variant.backend_repr = abi;1044                    // Also need to bump up the size, so that the entire value fits in here.1045                    variant.size = cmp::max(variant.size, size);1046                }1047            }1048        }10491050        let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);10511052        let tagged_layout = LayoutData {1053            variants: Variants::Multiple {1054                tag,1055                tag_encoding: TagEncoding::Direct,1056                tag_field: FieldIdx::new(0),1057                variants: layout_variants,1058            },1059            fields: FieldsShape::Arbitrary {1060                offsets: [Size::ZERO].into(),1061                in_memory_order: [FieldIdx::new(0)].into(),1062            },1063            largest_niche,1064            uninhabited,1065            backend_repr: abi,1066            align: AbiAlign::new(align),1067            size,1068            max_repr_align,1069            unadjusted_abi_align,1070            randomization_seed: combined_seed,1071        };10721073        let best_layout = match (tagged_layout, niche_filling_layout) {1074            (tl, Some(nl)) => {1075                // Pick the smaller layout; otherwise,1076                // pick the layout with the larger niche; otherwise,1077                // pick tagged as it has simpler codegen.1078                use cmp::Ordering::*;1079                let niche_size = |l: &LayoutData<FieldIdx, VariantIdx>| {1080                    l.largest_niche.map_or(0, |n| n.available(dl))1081                };1082                match (tl.size.cmp(&nl.size), niche_size(&tl).cmp(&niche_size(&nl))) {1083                    (Greater, _) => nl,1084                    (Equal, Less) => nl,1085                    _ => tl,1086                }1087            }1088            (tl, None) => tl,1089        };10901091        Ok(best_layout)1092    }10931094    fn univariant_biased<1095        'a,1096        FieldIdx: Idx,1097        VariantIdx: Idx,1098        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug + Copy,1099    >(1100        &self,1101        fields: &IndexSlice<FieldIdx, F>,1102        repr: &ReprOptions,1103        kind: StructKind,1104        niche_bias: NicheBias,1105    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {1106        let dl = self.cx.data_layout();1107        let pack = repr.pack;1108        let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };1109        let mut max_repr_align = repr.align;1110        let mut in_memory_order: IndexVec<u32, FieldIdx> = fields.indices().collect();1111        let optimize_field_order = !repr.inhibit_struct_field_reordering();1112        let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };1113        let optimizing = &mut in_memory_order.raw[..end];1114        let fields_excluding_tail = &fields.raw[..end];1115        // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts.1116        let field_seed = fields_excluding_tail1117            .iter()1118            .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed));11191120        if optimize_field_order && fields.len() > 1 {1121            // If `-Z randomize-layout` was enabled for the type definition we can shuffle1122            // the field ordering to try and catch some code making assumptions about layouts1123            // we don't guarantee.1124            if repr.can_randomize_type_layout() && cfg!(feature = "randomize") {1125                #[cfg(feature = "randomize")]1126                {1127                    use rand::SeedableRng;1128                    use rand::seq::SliceRandom;1129                    // `ReprOptions.field_shuffle_seed` is a deterministic seed we can use to randomize field1130                    // ordering.1131                    let mut rng = rand_xoshiro::Xoshiro128StarStar::seed_from_u64(1132                        field_seed.wrapping_add(repr.field_shuffle_seed).as_u64(),1133                    );11341135                    // Shuffle the ordering of the fields.1136                    optimizing.shuffle(&mut rng);1137                }1138                // Otherwise we just leave things alone and actually optimize the type's fields1139            } else {1140                // To allow unsizing `&Foo<Type>` -> `&Foo<dyn Trait>`, the layout of the struct must1141                // not depend on the layout of the tail.1142                let max_field_align =1143                    fields_excluding_tail.iter().map(|f| f.align.bytes()).max().unwrap_or(1);1144                let largest_niche_size = fields_excluding_tail1145                    .iter()1146                    .filter_map(|f| f.largest_niche)1147                    .map(|n| n.available(dl))1148                    .max()1149                    .unwrap_or(0);11501151                // Calculates a sort key to group fields by their alignment or possibly some1152                // size-derived pseudo-alignment.1153                let alignment_group_key = |layout: &F| {1154                    // The two branches here return values that cannot be meaningfully compared with1155                    // each other. However, we know that consistently for all executions of1156                    // `alignment_group_key`, one or the other branch will be taken, so this is okay.1157                    if let Some(pack) = pack {1158                        // Return the packed alignment in bytes.1159                        layout.align.abi.min(pack).bytes()1160                    } else {1161                        // Returns `log2(effective-align)`. The calculation assumes that size is an1162                        // integer multiple of align, except for ZSTs.1163                        let align = layout.align.bytes();1164                        let size = layout.size.bytes();1165                        let niche_size = layout.largest_niche.map(|n| n.available(dl)).unwrap_or(0);1166                        // Group [u8; 4] with align-4 or [u8; 6] with align-2 fields.1167                        let size_as_align = align.max(size).trailing_zeros();1168                        let size_as_align = if largest_niche_size > 0 {1169                            match niche_bias {1170                                // Given `A(u8, [u8; 16])` and `B(bool, [u8; 16])` we want to bump the1171                                // array to the front in the first case (for aligned loads) but keep1172                                // the bool in front in the second case for its niches.1173                                NicheBias::Start => {1174                                    max_field_align.trailing_zeros().min(size_as_align)1175                                }1176                                // When moving niches towards the end of the struct then for1177                                // A((u8, u8, u8, bool), (u8, bool, u8)) we want to keep the first tuple1178                                // in the align-1 group because its bool can be moved closer to the end.1179                                NicheBias::End if niche_size == largest_niche_size => {1180                                    align.trailing_zeros()1181                                }1182                                NicheBias::End => size_as_align,1183                            }1184                        } else {1185                            size_as_align1186                        };1187                        size_as_align as u641188                    }1189                };11901191                match kind {1192                    StructKind::AlwaysSized | StructKind::MaybeUnsized => {1193                        // Currently `LayoutData` only exposes a single niche so sorting is usually1194                        // sufficient to get one niche into the preferred position. If it ever1195                        // supported multiple niches then a more advanced pick-and-pack approach could1196                        // provide better results. But even for the single-niche cache it's not1197                        // optimal. E.g. for A(u32, (bool, u8), u16) it would be possible to move the1198                        // bool to the front but it would require packing the tuple together with the1199                        // u16 to build a 4-byte group so that the u32 can be placed after it without1200                        // padding. This kind of packing can't be achieved by sorting.1201                        optimizing.sort_by_key(|&x| {1202                            let f = &fields[x];1203                            let field_size = f.size.bytes();1204                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));1205                            let niche_size_key = match niche_bias {1206                                // large niche first1207                                NicheBias::Start => !niche_size,1208                                // large niche last1209                                NicheBias::End => niche_size,1210                            };1211                            let inner_niche_offset_key = match niche_bias {1212                                NicheBias::Start => f.largest_niche.map_or(0, |n| n.offset.bytes()),1213                                NicheBias::End => f.largest_niche.map_or(0, |n| {1214                                    !(field_size - n.value.size(dl).bytes() - n.offset.bytes())1215                                }),1216                            };12171218                            (1219                                // Then place largest alignments first.1220                                cmp::Reverse(alignment_group_key(f)),1221                                // Then prioritize niche placement within alignment group according to1222                                // `niche_bias_start`.1223                                niche_size_key,1224                                // Then among fields with equally-sized niches prefer the ones1225                                // closer to the start/end of the field.1226                                inner_niche_offset_key,1227                            )1228                        });1229                    }12301231                    StructKind::Prefixed(..) => {1232                        // Sort in ascending alignment so that the layout stays optimal1233                        // regardless of the prefix.1234                        // And put the largest niche in an alignment group at the end1235                        // so it can be used as discriminant in jagged enums1236                        optimizing.sort_by_key(|&x| {1237                            let f = &fields[x];1238                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));1239                            (alignment_group_key(f), niche_size)1240                        });1241                    }1242                }12431244                // FIXME(Kixiron): We can always shuffle fields within a given alignment class1245                //                 regardless of the status of `-Z randomize-layout`1246            }1247        }1248        // in_memory_order holds field indices by increasing memory offset.1249        // That is, if field 5 has offset 0, the first element of in_memory_order is 5.1250        // We now write field offsets to the corresponding offset slot;1251        // field 5 with offset 0 puts 0 in offsets[5].1252        let mut unsized_field = None::<&F>;1253        let mut offsets = IndexVec::from_elem(Size::ZERO, fields);1254        let mut offset = Size::ZERO;1255        let mut largest_niche = None;1256        let mut largest_niche_available = 0;1257        if let StructKind::Prefixed(prefix_size, prefix_align) = kind {1258            let prefix_align =1259                if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };1260            align = align.max(prefix_align);1261            offset = prefix_size.align_to(prefix_align);1262        }1263        for &i in &in_memory_order {1264            let field = &fields[i];1265            if let Some(unsized_field) = unsized_field {1266                return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field));1267            }12681269            if field.is_unsized() {1270                if let StructKind::MaybeUnsized = kind {1271                    unsized_field = Some(field);1272                } else {1273                    return Err(LayoutCalculatorError::UnexpectedUnsized(*field));1274                }1275            }12761277            // Invariant: offset < dl.obj_size_bound() <= 1<<611278            let field_align = if let Some(pack) = pack {1279                field.align.min(AbiAlign::new(pack))1280            } else {1281                field.align1282            };1283            offset = offset.align_to(field_align.abi);1284            align = align.max(field_align.abi);1285            max_repr_align = max_repr_align.max(field.max_repr_align);12861287            debug!("univariant offset: {:?} field: {:#?}", offset, field);1288            offsets[i] = offset;12891290            if let Some(mut niche) = field.largest_niche {1291                let available = niche.available(dl);1292                // Pick up larger niches.1293                let prefer_new_niche = match niche_bias {1294                    NicheBias::Start => available > largest_niche_available,1295                    // if there are several niches of the same size then pick the last one1296                    NicheBias::End => available >= largest_niche_available,1297                };1298                if prefer_new_niche {1299                    largest_niche_available = available;1300                    niche.offset += offset;1301                    largest_niche = Some(niche);1302                }1303            }13041305            offset =1306                offset.checked_add(field.size, dl).ok_or(LayoutCalculatorError::SizeOverflow)?;1307        }13081309        // The unadjusted ABI alignment does not include repr(align), but does include repr(pack).1310        // See documentation on `LayoutData::unadjusted_abi_align`.1311        let unadjusted_abi_align = align;1312        if let Some(repr_align) = repr.align {1313            align = align.max(repr_align);1314        }1315        // `align` must not be modified after this point, or `unadjusted_abi_align` could be inaccurate.1316        let align = align;13171318        debug!("univariant min_size: {:?}", offset);1319        let min_size = offset;1320        let size = min_size.align_to(align);1321        // FIXME(oli-obk): deduplicate and harden these checks1322        if size.bytes() >= dl.obj_size_bound() {1323            return Err(LayoutCalculatorError::SizeOverflow);1324        }1325        let mut layout_of_single_non_zst_field = None;1326        let sized = unsized_field.is_none();1327        let mut abi = BackendRepr::Memory { sized };13281329        let optimize_abi = !repr.inhibit_newtype_abi_optimization();13301331        // Try to make this a Scalar/ScalarPair.1332        if sized && size.bytes() > 0 {1333            // We skip *all* ZST here and later check if we are good in terms of alignment.1334            // This lets us handle some cases involving aligned ZST.1335            let mut non_zst_fields = fields.iter_enumerated().filter(|&(_, f)| !f.is_zst());13361337            match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {1338                // We have exactly one non-ZST field.1339                (Some((i, field)), None, None) => {1340                    layout_of_single_non_zst_field = Some(field);13411342                    // Field fills the struct and it has a scalar or scalar pair ABI.1343                    if offsets[i].bytes() == 0 && align == field.align.abi && size == field.size {1344                        match field.backend_repr {1345                            // For plain scalars, or vectors of them, we can't unpack1346                            // newtypes for `#[repr(C)]`, as that affects C ABIs.1347                            BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. }1348                                if optimize_abi =>1349                            {1350                                abi = field.backend_repr;1351                            }1352                            // But scalar pairs are Rust-specific and get1353                            // treated as aggregates by C ABIs anyway.1354                            BackendRepr::ScalarPair(..) => {1355                                abi = field.backend_repr;1356                            }1357                            _ => {}1358                        }1359                    }1360                }13611362                // Two non-ZST fields, and they're both scalars.1363                (Some((i, a)), Some((j, b)), None) => {1364                    match (a.backend_repr, b.backend_repr) {1365                        (BackendRepr::Scalar(a), BackendRepr::Scalar(b)) => {1366                            // Order by the memory placement, not source order.1367                            let ((i, a), (j, b)) = if offsets[i] < offsets[j] {1368                                ((i, a), (j, b))1369                            } else {1370                                ((j, b), (i, a))1371                            };1372                            let pair =1373                                LayoutData::<FieldIdx, VariantIdx>::scalar_pair(&self.cx, a, b);1374                            let pair_offsets = match pair.fields {1375                                FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => {1376                                    assert_eq!(1377                                        in_memory_order.raw,1378                                        [FieldIdx::new(0), FieldIdx::new(1)]1379                                    );1380                                    offsets1381                                }1382                                FieldsShape::Primitive1383                                | FieldsShape::Array { .. }1384                                | FieldsShape::Union(..) => {1385                                    panic!("encountered a non-arbitrary layout during enum layout")1386                                }1387                            };1388                            if offsets[i] == pair_offsets[FieldIdx::new(0)]1389                                && offsets[j] == pair_offsets[FieldIdx::new(1)]1390                                && align == pair.align.abi1391                                && size == pair.size1392                            {1393                                // We can use `ScalarPair` only when it matches our1394                                // already computed layout (including `#[repr(C)]`).1395                                abi = pair.backend_repr;1396                            }1397                        }1398                        _ => {}1399                    }1400                }14011402                _ => {}1403            }1404        }1405        let uninhabited = fields.iter().any(|f| f.is_uninhabited());14061407        let unadjusted_abi_align = if repr.transparent() {1408            match layout_of_single_non_zst_field {1409                Some(l) => l.unadjusted_abi_align,1410                None => {1411                    // `repr(transparent)` with all ZST fields.1412                    align1413                }1414            }1415        } else {1416            unadjusted_abi_align1417        };14181419        let seed = field_seed.wrapping_add(repr.field_shuffle_seed);14201421        Ok(LayoutData {1422            variants: Variants::Single { index: VariantIdx::new(0) },1423            fields: FieldsShape::Arbitrary { offsets, in_memory_order },1424            backend_repr: abi,1425            largest_niche,1426            uninhabited,1427            align: AbiAlign::new(align),1428            size,1429            max_repr_align,1430            unadjusted_abi_align,1431            randomization_seed: seed,1432        })1433    }14341435    fn format_field_niches<1436        'a,1437        FieldIdx: Idx,1438        VariantIdx: Idx,1439        F: Deref<Target = &'a LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,1440    >(1441        &self,1442        layout: &LayoutData<FieldIdx, VariantIdx>,1443        fields: &IndexSlice<FieldIdx, F>,1444    ) -> String {1445        let dl = self.cx.data_layout();1446        let mut s = String::new();1447        for i in layout.fields.index_by_increasing_offset() {1448            let offset = layout.fields.offset(i);1449            let f = &fields[FieldIdx::new(i)];1450            write!(s, "[o{}a{}s{}", offset.bytes(), f.align.bytes(), f.size.bytes()).unwrap();1451            if let Some(n) = f.largest_niche {1452                write!(1453                    s,1454                    " n{}b{}s{}",1455                    n.offset.bytes(),1456                    n.available(dl).ilog2(),1457                    n.value.size(dl).bytes()1458                )1459                .unwrap();1460            }1461            write!(s, "] ").unwrap();1462        }1463        s1464    }1465}14661467enum SimdVectorKind {1468    /// `#[rustc_scalable_vector]`1469    Scalable(NumScalableVectors),1470    /// `#[repr(simd, packed)]`1471    PackedFixed,1472    /// `#[repr(simd)]`1473    Fixed,1474}14751476fn vector_type_layout<FieldIdx, VariantIdx, F>(1477    kind: SimdVectorKind,1478    dl: &TargetDataLayout,1479    element: F,1480    count: u64,1481) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>1482where1483    FieldIdx: Idx,1484    VariantIdx: Idx,1485    F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,1486{1487    let elt = element.as_ref();1488    if count == 0 {1489        return Err(LayoutCalculatorError::ZeroLengthSimdType);1490    } else if count > crate::MAX_SIMD_LANES {1491        return Err(LayoutCalculatorError::OversizedSimdType { max_lanes: crate::MAX_SIMD_LANES });1492    }14931494    let BackendRepr::Scalar(element) = elt.backend_repr else {1495        return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));1496    };14971498    // Compute the size and alignment of the vector1499    let size =1500        elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;1501    let (repr, size, align) = match kind {1502        SimdVectorKind::Scalable(number_of_vectors) => (1503            BackendRepr::SimdScalableVector { element, count, number_of_vectors },1504            size.checked_mul(number_of_vectors.0 as u64, dl)1505                .ok_or_else(|| LayoutCalculatorError::SizeOverflow)?,1506            dl.llvmlike_vector_align(size),1507        ),1508        // Non-power-of-two vectors have padding up to the next power-of-two.1509        // If we're a packed repr, remove the padding while keeping the alignment as close1510        // to a vector as possible.1511        SimdVectorKind::PackedFixed if !count.is_power_of_two() => {1512            (BackendRepr::Memory { sized: true }, size, Align::max_aligned_factor(size))1513        }1514        SimdVectorKind::PackedFixed | SimdVectorKind::Fixed => {1515            (BackendRepr::SimdVector { element, count }, size, dl.llvmlike_vector_align(size))1516        }1517    };1518    let size = size.align_to(align);15191520    Ok(LayoutData {1521        variants: Variants::Single { index: VariantIdx::new(0) },1522        fields: FieldsShape::Arbitrary {1523            offsets: [Size::ZERO].into(),1524            in_memory_order: [FieldIdx::new(0)].into(),1525        },1526        backend_repr: repr,1527        largest_niche: elt.largest_niche,1528        uninhabited: false,1529        size,1530        align: AbiAlign::new(align),1531        max_repr_align: None,1532        unadjusted_abi_align: elt.align.abi,1533        randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),1534    })1535}