1//! Interpret intrinsics, lang items and `extern "C"` wellknown functions which their implementation2//! is not available.3//!4use std::cmp::{self, Ordering};56use hir_def::{attrs::AttrFlags, signatures::FunctionSignature};7use hir_expand::name::Name;8use intern::sym;9use rustc_type_ir::inherent::{AdtDef, GenericArgs as _, IntoKind, SliceLike, Ty as _};10use stdx::never;1112use crate::{13    display::DisplayTarget,14    drop::{DropGlue, has_drop_glue},15    mir::eval::{16        Address, AdtId, Arc, Evaluator, FunctionId, GenericArgs, HasModule, HirDisplay, Interval,17        IntervalAndTy, IntervalOrOwned, ItemContainerId, Layout, Locals, Lookup, MirEvalError,18        MirSpan, Mutability, Result, Ty, TyKind, from_bytes, not_supported, pad16,19    },20    next_solver::Region,21};2223mod simd;2425#[derive(Debug, Clone, Copy, PartialEq, Eq)]26enum EvalLangItem {27    BeginPanic,28    SliceLen,29    DropInPlace,30}3132impl<'db> Evaluator<'db> {33    pub(super) fn detect_and_exec_special_function(34        &mut self,35        def: FunctionId,36        args: &[IntervalAndTy<'db>],37        generic_args: GenericArgs<'db>,38        locals: &Locals,39        destination: Interval,40        span: MirSpan,41    ) -> Result<'db, bool> {42        if self.not_special_fn_cache.borrow().contains(&def) {43            return Ok(false);44        }4546        let function_data = FunctionSignature::of(self.db, def);47        let attrs = AttrFlags::query(self.db, def.into());48        let is_intrinsic = FunctionSignature::is_intrinsic(self.db, def);4950        if is_intrinsic {51            return self.exec_intrinsic(52                function_data.name.as_str(),53                args,54                generic_args,55                destination,56                locals,57                span,58                !function_data.has_body()59                    || attrs.contains(AttrFlags::RUSTC_INTRINSIC_MUST_BE_OVERRIDDEN),60            );61        }62        let is_extern_c = match def.lookup(self.db).container {63            hir_def::ItemContainerId::ExternBlockId(block) => block.abi(self.db) == Some(sym::C),64            _ => false,65        };66        if is_extern_c {67            return self68                .exec_extern_c(69                    function_data.name.as_str(),70                    args,71                    generic_args,72                    destination,73                    locals,74                    span,75                )76                .map(|()| true);77        }7879        if attrs.intersects(80            AttrFlags::RUSTC_ALLOCATOR81                | AttrFlags::RUSTC_DEALLOCATOR82                | AttrFlags::RUSTC_REALLOCATOR83                | AttrFlags::RUSTC_ALLOCATOR_ZEROED,84        ) {85            self.exec_alloc_fn(attrs, args, destination)?;86            return Ok(true);87        }88        if let Some(it) = self.detect_lang_function(def) {89            let result = self.exec_lang_item(it, generic_args, args, locals, span)?;90            destination.write_from_bytes(self, &result)?;91            return Ok(true);92        }93        if let ItemContainerId::TraitId(t) = def.lookup(self.db).container94            && Some(t) == self.lang_items().Clone95        {96            let [self_ty] = generic_args.as_slice() else {97                not_supported!("wrong generic arg count for clone");98            };99            let Some(self_ty) = self_ty.ty() else {100                not_supported!("wrong generic arg kind for clone");101            };102            // Clone has special impls for tuples and function pointers103            if matches!(self_ty.kind(), TyKind::FnPtr(..) | TyKind::Tuple(..) | TyKind::Closure(..))104            {105                self.exec_clone(def, args, self_ty, locals, destination, span)?;106                return Ok(true);107            }108            // Return early to prevent caching clone as non special fn.109            return Ok(false);110        }111        self.not_special_fn_cache.borrow_mut().insert(def);112        Ok(false)113    }114115    pub(super) fn detect_and_redirect_special_function(116        &mut self,117        def: FunctionId,118    ) -> Result<'db, Option<FunctionId>> {119        // `PanicFmt` is redirected to `ConstPanicFmt`120        if Some(def) == self.lang_items().PanicFmt {121            let Some(const_panic_fmt) = self.lang_items().ConstPanicFmt else {122                not_supported!("const_panic_fmt lang item not found or not a function");123            };124            return Ok(Some(const_panic_fmt));125        }126        Ok(None)127    }128129    /// Clone has special impls for tuples and function pointers130    fn exec_clone(131        &mut self,132        def: FunctionId,133        args: &[IntervalAndTy<'db>],134        self_ty: Ty<'db>,135        locals: &Locals,136        destination: Interval,137        span: MirSpan,138    ) -> Result<'db, ()> {139        match self_ty.kind() {140            TyKind::FnPtr(..) => {141                let [arg] = args else {142                    not_supported!("wrong arg count for clone");143                };144                let addr = Address::from_bytes(arg.get(self)?)?;145                return destination146                    .write_from_interval(self, Interval { addr, size: destination.size });147            }148            TyKind::Closure(_, closure_args) => self.exec_clone(149                def,150                args,151                closure_args.as_closure().tupled_upvars_ty(),152                locals,153                destination,154                span,155            )?,156            TyKind::Tuple(subst) => {157                let [arg] = args else {158                    not_supported!("wrong arg count for clone");159                };160                let addr = Address::from_bytes(arg.get(self)?)?;161                let layout = self.layout(self_ty)?;162                self.exec_clone_for_fields(163                    subst.iter(),164                    layout,165                    addr,166                    def,167                    locals,168                    destination,169                    span,170                )?;171            }172            _ => {173                self.exec_fn_with_args(174                    def,175                    args,176                    GenericArgs::new_from_slice(&[self_ty.into()]),177                    locals,178                    destination,179                    None,180                    span,181                )?;182            }183        }184        Ok(())185    }186187    fn exec_clone_for_fields(188        &mut self,189        ty_iter: impl Iterator<Item = Ty<'db>>,190        layout: Arc<Layout>,191        addr: Address,192        def: FunctionId,193        locals: &Locals,194        destination: Interval,195        span: MirSpan,196    ) -> Result<'db, ()> {197        for (i, ty) in ty_iter.enumerate() {198            let size = self.layout(ty)?.size.bytes_usize();199            let tmp = self.heap_allocate(self.ptr_size(), self.ptr_size())?;200            let arg = IntervalAndTy {201                interval: Interval { addr: tmp, size: self.ptr_size() },202                ty: Ty::new_ref(203                    self.interner(),204                    Region::error(self.interner()),205                    ty,206                    Mutability::Not,207                ),208            };209            let offset = layout.fields.offset(i).bytes_usize();210            self.write_memory(tmp, &addr.offset(offset).to_bytes())?;211            self.exec_clone(212                def,213                &[arg],214                ty,215                locals,216                destination.slice(offset..offset + size),217                span,218            )?;219        }220        Ok(())221    }222223    fn exec_alloc_fn(224        &mut self,225        alloc_fn: AttrFlags,226        args: &[IntervalAndTy<'db>],227        destination: Interval,228    ) -> Result<'db, ()> {229        match alloc_fn {230            _ if alloc_fn231                .intersects(AttrFlags::RUSTC_ALLOCATOR_ZEROED | AttrFlags::RUSTC_ALLOCATOR) =>232            {233                let [size, align] = args else {234                    return Err(MirEvalError::InternalError(235                        "rustc_allocator args are not provided".into(),236                    ));237                };238                let size = from_bytes!(usize, size.get(self)?);239                let align = from_bytes!(usize, align.get(self)?);240                let result = self.heap_allocate(size, align)?;241                destination.write_from_bytes(self, &result.to_bytes())?;242            }243            _ if alloc_fn.contains(AttrFlags::RUSTC_DEALLOCATOR) => { /* no-op for now */ }244            _ if alloc_fn.contains(AttrFlags::RUSTC_REALLOCATOR) => {245                let [ptr, old_size, align, new_size] = args else {246                    return Err(MirEvalError::InternalError(247                        "rustc_allocator args are not provided".into(),248                    ));249                };250                let old_size = from_bytes!(usize, old_size.get(self)?);251                let new_size = from_bytes!(usize, new_size.get(self)?);252                if old_size >= new_size {253                    destination.write_from_interval(self, ptr.interval)?;254                } else {255                    let ptr = Address::from_bytes(ptr.get(self)?)?;256                    let align = from_bytes!(usize, align.get(self)?);257                    let result = self.heap_allocate(new_size, align)?;258                    Interval { addr: result, size: old_size }259                        .write_from_interval(self, Interval { addr: ptr, size: old_size })?;260                    destination.write_from_bytes(self, &result.to_bytes())?;261                }262            }263            _ => not_supported!("unknown alloc function"),264        }265        Ok(())266    }267268    fn detect_lang_function(&self, def: FunctionId) -> Option<EvalLangItem> {269        use EvalLangItem::*;270        let lang_items = self.lang_items();271        let attrs = AttrFlags::query(self.db, def.into());272273        if attrs.contains(AttrFlags::RUSTC_CONST_PANIC_STR) {274            // `#[rustc_const_panic_str]` is treated like `lang = "begin_panic"` by rustc CTFE.275            return Some(BeginPanic);276        }277278        // We want to execute these functions with special logic279        // `PanicFmt` is not detected here as it's redirected later.280        if let Some((_, candidate)) = [281            (lang_items.BeginPanic, BeginPanic),282            (lang_items.SliceLen, SliceLen),283            (lang_items.DropInPlace, DropInPlace),284        ]285        .iter()286        .find(|&(candidate, _)| candidate == Some(def))287        {288            return Some(candidate);289        }290291        None292    }293294    fn exec_lang_item(295        &mut self,296        it: EvalLangItem,297        generic_args: GenericArgs<'db>,298        args: &[IntervalAndTy<'db>],299        locals: &Locals,300        span: MirSpan,301    ) -> Result<'db, Vec<u8>> {302        use EvalLangItem::*;303        let mut args = args.iter();304        match it {305            BeginPanic => {306                let mut arg = args307                    .next()308                    .ok_or(MirEvalError::InternalError(309                        "argument of BeginPanic is not provided".into(),310                    ))?311                    .clone();312                while let TyKind::Ref(_, ty, _) = arg.ty.kind() {313                    if ty.is_str() {314                        let (pointee, metadata) = arg.interval.get(self)?.split_at(self.ptr_size());315                        let len = from_bytes!(usize, metadata);316317                        return {318                            Err(MirEvalError::Panic(319                                std::str::from_utf8(320                                    self.read_memory(Address::from_bytes(pointee)?, len)?,321                                )322                                .unwrap()323                                .to_owned(),324                            ))325                        };326                    }327                    let size = self.size_of_sized(ty, locals, "begin panic arg")?;328                    let pointee = arg.interval.get(self)?;329                    arg = IntervalAndTy {330                        interval: Interval::new(Address::from_bytes(pointee)?, size),331                        ty,332                    };333                }334                Err(MirEvalError::Panic(format!("unknown-panic-payload: {:?}", arg.ty.kind())))335            }336            SliceLen => {337                let arg = args.next().ok_or(MirEvalError::InternalError(338                    "argument of <[T]>::len() is not provided".into(),339                ))?;340                let arg = arg.get(self)?;341                let ptr_size = arg.len() / 2;342                Ok(arg[ptr_size..].into())343            }344            DropInPlace => {345                let ty = generic_args.as_slice().first().and_then(|it| it.ty()).ok_or(346                    MirEvalError::InternalError(347                        "generic argument of drop_in_place is not provided".into(),348                    ),349                )?;350                let arg = args.next().ok_or(MirEvalError::InternalError(351                    "argument of drop_in_place is not provided".into(),352                ))?;353                let arg = arg.interval.get(self)?.to_owned();354                self.run_drop_glue_deep(355                    ty,356                    locals,357                    Address::from_bytes(&arg[0..self.ptr_size()])?,358                    &arg[self.ptr_size()..],359                    span,360                )?;361                Ok(vec![])362            }363        }364    }365366    fn exec_syscall(367        &mut self,368        id: i64,369        args: &[IntervalAndTy<'db>],370        destination: Interval,371        _locals: &Locals,372        _span: MirSpan,373    ) -> Result<'db, ()> {374        match id {375            318 => {376                // SYS_getrandom377                let [buf, len, _flags] = args else {378                    return Err(MirEvalError::InternalError(379                        "SYS_getrandom args are not provided".into(),380                    ));381                };382                let addr = Address::from_bytes(buf.get(self)?)?;383                let size = from_bytes!(usize, len.get(self)?);384                for i in 0..size {385                    let rand_byte = self.random_state.rand_u64() as u8;386                    self.write_memory(addr.offset(i), &[rand_byte])?;387                }388                destination.write_from_interval(self, len.interval)389            }390            _ => {391                not_supported!("Unknown syscall id {id:?}")392            }393        }394    }395396    fn exec_extern_c(397        &mut self,398        as_str: &str,399        args: &[IntervalAndTy<'db>],400        _generic_args: GenericArgs<'db>,401        destination: Interval,402        locals: &Locals,403        span: MirSpan,404    ) -> Result<'db, ()> {405        match as_str {406            "memcmp" => {407                let [ptr1, ptr2, size] = args else {408                    return Err(MirEvalError::InternalError("memcmp args are not provided".into()));409                };410                let addr1 = Address::from_bytes(ptr1.get(self)?)?;411                let addr2 = Address::from_bytes(ptr2.get(self)?)?;412                let size = from_bytes!(usize, size.get(self)?);413                let slice1 = self.read_memory(addr1, size)?;414                let slice2 = self.read_memory(addr2, size)?;415                let r: i128 = match slice1.cmp(slice2) {416                    cmp::Ordering::Less => -1,417                    cmp::Ordering::Equal => 0,418                    cmp::Ordering::Greater => 1,419                };420                destination.write_from_bytes(self, &r.to_le_bytes()[..destination.size])421            }422            "write" => {423                let [fd, ptr, len] = args else {424                    return Err(MirEvalError::InternalError(425                        "libc::write args are not provided".into(),426                    ));427                };428                let fd = u128::from_le_bytes(pad16(fd.get(self)?, false));429                let interval = Interval {430                    addr: Address::from_bytes(ptr.get(self)?)?,431                    size: from_bytes!(usize, len.get(self)?),432                };433                match fd {434                    1 => {435                        self.write_to_stdout(interval)?;436                    }437                    2 => {438                        self.write_to_stderr(interval)?;439                    }440                    _ => not_supported!("write to arbitrary file descriptor"),441                }442                destination.write_from_interval(self, len.interval)?;443                Ok(())444            }445            "pthread_key_create" => {446                let key = self.thread_local_storage.create_key();447                let Some(arg0) = args.first() else {448                    return Err(MirEvalError::InternalError(449                        "pthread_key_create arg0 is not provided".into(),450                    ));451                };452                let arg0_addr = Address::from_bytes(arg0.get(self)?)?;453                let key_ty = if let Some((ty, ..)) = arg0.ty.as_reference_or_ptr() {454                    ty455                } else {456                    return Err(MirEvalError::InternalError(457                        "pthread_key_create arg0 is not a pointer".into(),458                    ));459                };460                let arg0_interval = Interval::new(461                    arg0_addr,462                    self.size_of_sized(key_ty, locals, "pthread_key_create key arg")?,463                );464                arg0_interval.write_from_bytes(self, &key.to_le_bytes()[0..arg0_interval.size])?;465                // return 0 as success466                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;467                Ok(())468            }469            "pthread_getspecific" => {470                let Some(arg0) = args.first() else {471                    return Err(MirEvalError::InternalError(472                        "pthread_getspecific arg0 is not provided".into(),473                    ));474                };475                let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]);476                let value = self.thread_local_storage.get_key(key)?;477                destination.write_from_bytes(self, &value.to_le_bytes()[0..destination.size])?;478                Ok(())479            }480            "pthread_setspecific" => {481                let Some(arg0) = args.first() else {482                    return Err(MirEvalError::InternalError(483                        "pthread_setspecific arg0 is not provided".into(),484                    ));485                };486                let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]);487                let Some(arg1) = args.get(1) else {488                    return Err(MirEvalError::InternalError(489                        "pthread_setspecific arg1 is not provided".into(),490                    ));491                };492                let value = from_bytes!(u128, pad16(arg1.get(self)?, false));493                self.thread_local_storage.set_key(key, value)?;494                // return 0 as success495                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;496                Ok(())497            }498            "pthread_key_delete" => {499                // we ignore this currently500                // return 0 as success501                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;502                Ok(())503            }504            "syscall" => {505                let Some((id, rest)) = args.split_first() else {506                    return Err(MirEvalError::InternalError("syscall arg1 is not provided".into()));507                };508                let id = from_bytes!(i64, id.get(self)?);509                self.exec_syscall(id, rest, destination, locals, span)510            }511            "sched_getaffinity" => {512                let [_pid, _set_size, set] = args else {513                    return Err(MirEvalError::InternalError(514                        "sched_getaffinity args are not provided".into(),515                    ));516                };517                let set = Address::from_bytes(set.get(self)?)?;518                // Only enable core 0 (we are single threaded anyway), which is bitset 0x0000001519                self.write_memory(set, &[1])?;520                // return 0 as success521                self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);522                Ok(())523            }524            "getenv" => {525                let [name] = args else {526                    return Err(MirEvalError::InternalError("getenv args are not provided".into()));527                };528                let mut name_buf = vec![];529                let name = {530                    let mut index = Address::from_bytes(name.get(self)?)?;531                    loop {532                        let byte = self.read_memory(index, 1)?[0];533                        index = index.offset(1);534                        if byte == 0 {535                            break;536                        }537                        name_buf.push(byte);538                    }539                    String::from_utf8_lossy(&name_buf)540                };541                let value = self.crate_id.env(self.db).get(&name);542                match value {543                    None => {544                        // Write null as fail545                        self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);546                    }547                    Some(mut value) => {548                        value.push('\0');549                        let addr = self.heap_allocate(value.len(), 1)?;550                        self.write_memory(addr, value.as_bytes())?;551                        self.write_memory(destination.addr, &addr.to_bytes())?;552                    }553                }554                Ok(())555            }556            _ => not_supported!("unknown external function {as_str}"),557        }558    }559560    fn exec_intrinsic(561        &mut self,562        name: &str,563        args: &[IntervalAndTy<'db>],564        generic_args: GenericArgs<'db>,565        destination: Interval,566        locals: &Locals,567        span: MirSpan,568        needs_override: bool,569    ) -> Result<'db, bool> {570        if let Some(name) = name.strip_prefix("atomic_") {571            return self572                .exec_atomic_intrinsic(name, args, generic_args, destination, locals, span)573                .map(|()| true);574        }575        if let Some(name) = name.strip_prefix("simd_") {576            return self577                .exec_simd_intrinsic(name, args, generic_args, destination, locals, span)578                .map(|()| true);579        }580        // FIXME(#17451): Add `f16` and `f128` intrinsics.581        if let Some(name) = name.strip_suffix("f64") {582            let result = match name {583                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"584                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {585                    let [arg] = args else {586                        return Err(MirEvalError::InternalError(587                            "f64 intrinsic signature doesn't match fn (f64) -> f64".into(),588                        ));589                    };590                    let arg = from_bytes!(f64, arg.get(self)?);591                    match name {592                        "sqrt" => arg.sqrt(),593                        "sin" => arg.sin(),594                        "cos" => arg.cos(),595                        "exp" => arg.exp(),596                        "exp2" => arg.exp2(),597                        "log" => arg.ln(),598                        "log10" => arg.log10(),599                        "log2" => arg.log2(),600                        "fabs" => arg.abs(),601                        "floor" => arg.floor(),602                        "ceil" => arg.ceil(),603                        "trunc" => arg.trunc(),604                        // FIXME: these rounds should be different, but only `.round()` is stable now.605                        "rint" => arg.round(),606                        "nearbyint" => arg.round(),607                        "round" => arg.round(),608                        "roundeven" => arg.round(),609                        _ => unreachable!(),610                    }611                }612                "pow" | "minnum" | "maxnum" | "copysign" => {613                    let [arg1, arg2] = args else {614                        return Err(MirEvalError::InternalError(615                            "f64 intrinsic signature doesn't match fn (f64, f64) -> f64".into(),616                        ));617                    };618                    let arg1 = from_bytes!(f64, arg1.get(self)?);619                    let arg2 = from_bytes!(f64, arg2.get(self)?);620                    match name {621                        "pow" => arg1.powf(arg2),622                        "minnum" => arg1.min(arg2),623                        "maxnum" => arg1.max(arg2),624                        "copysign" => arg1.copysign(arg2),625                        _ => unreachable!(),626                    }627                }628                "powi" => {629                    let [arg1, arg2] = args else {630                        return Err(MirEvalError::InternalError(631                            "powif64 signature doesn't match fn (f64, i32) -> f64".into(),632                        ));633                    };634                    let arg1 = from_bytes!(f64, arg1.get(self)?);635                    let arg2 = from_bytes!(i32, arg2.get(self)?);636                    arg1.powi(arg2)637                }638                "fma" => {639                    let [arg1, arg2, arg3] = args else {640                        return Err(MirEvalError::InternalError(641                            "fmaf64 signature doesn't match fn (f64, f64, f64) -> f64".into(),642                        ));643                    };644                    let arg1 = from_bytes!(f64, arg1.get(self)?);645                    let arg2 = from_bytes!(f64, arg2.get(self)?);646                    let arg3 = from_bytes!(f64, arg3.get(self)?);647                    arg1.mul_add(arg2, arg3)648                }649                _ => not_supported!("unknown f64 intrinsic {name}"),650            };651            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);652        }653        if let Some(name) = name.strip_suffix("f32") {654            let result = match name {655                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"656                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {657                    let [arg] = args else {658                        return Err(MirEvalError::InternalError(659                            "f32 intrinsic signature doesn't match fn (f32) -> f32".into(),660                        ));661                    };662                    let arg = from_bytes!(f32, arg.get(self)?);663                    match name {664                        "sqrt" => arg.sqrt(),665                        "sin" => arg.sin(),666                        "cos" => arg.cos(),667                        "exp" => arg.exp(),668                        "exp2" => arg.exp2(),669                        "log" => arg.ln(),670                        "log10" => arg.log10(),671                        "log2" => arg.log2(),672                        "fabs" => arg.abs(),673                        "floor" => arg.floor(),674                        "ceil" => arg.ceil(),675                        "trunc" => arg.trunc(),676                        // FIXME: these rounds should be different, but only `.round()` is stable now.677                        "rint" => arg.round(),678                        "nearbyint" => arg.round(),679                        "round" => arg.round(),680                        "roundeven" => arg.round(),681                        _ => unreachable!(),682                    }683                }684                "pow" | "minnum" | "maxnum" | "copysign" => {685                    let [arg1, arg2] = args else {686                        return Err(MirEvalError::InternalError(687                            "f32 intrinsic signature doesn't match fn (f32, f32) -> f32".into(),688                        ));689                    };690                    let arg1 = from_bytes!(f32, arg1.get(self)?);691                    let arg2 = from_bytes!(f32, arg2.get(self)?);692                    match name {693                        "pow" => arg1.powf(arg2),694                        "minnum" => arg1.min(arg2),695                        "maxnum" => arg1.max(arg2),696                        "copysign" => arg1.copysign(arg2),697                        _ => unreachable!(),698                    }699                }700                "powi" => {701                    let [arg1, arg2] = args else {702                        return Err(MirEvalError::InternalError(703                            "powif32 signature doesn't match fn (f32, i32) -> f32".into(),704                        ));705                    };706                    let arg1 = from_bytes!(f32, arg1.get(self)?);707                    let arg2 = from_bytes!(i32, arg2.get(self)?);708                    arg1.powi(arg2)709                }710                "fma" => {711                    let [arg1, arg2, arg3] = args else {712                        return Err(MirEvalError::InternalError(713                            "fmaf32 signature doesn't match fn (f32, f32, f32) -> f32".into(),714                        ));715                    };716                    let arg1 = from_bytes!(f32, arg1.get(self)?);717                    let arg2 = from_bytes!(f32, arg2.get(self)?);718                    let arg3 = from_bytes!(f32, arg3.get(self)?);719                    arg1.mul_add(arg2, arg3)720                }721                _ => not_supported!("unknown f32 intrinsic {name}"),722            };723            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);724        }725        match name {726            "size_of" => {727                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {728                    return Err(MirEvalError::InternalError(729                        "size_of generic arg is not provided".into(),730                    ));731                };732                let size = self.size_of_sized(ty, locals, "size_of arg")?;733                destination.write_from_bytes(self, &size.to_le_bytes()[0..destination.size])734            }735            // FIXME: `min_align_of` was renamed to `align_of` in Rust 1.89736            // (https://github.com/rust-lang/rust/pull/142410)737            "min_align_of" | "align_of" => {738                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {739                    return Err(MirEvalError::InternalError(740                        "align_of generic arg is not provided".into(),741                    ));742                };743                let align = self.layout(ty)?.align.bytes();744                destination.write_from_bytes(self, &align.to_le_bytes()[0..destination.size])745            }746            "size_of_val" => {747                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {748                    return Err(MirEvalError::InternalError(749                        "size_of_val generic arg is not provided".into(),750                    ));751                };752                let [arg] = args else {753                    return Err(MirEvalError::InternalError(754                        "size_of_val args are not provided".into(),755                    ));756                };757                if let Some((size, _)) = self.size_align_of(ty, locals)? {758                    destination.write_from_bytes(self, &size.to_le_bytes())759                } else {760                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);761                    let (size, _) = self.size_align_of_unsized(ty, metadata, locals)?;762                    destination.write_from_bytes(self, &size.to_le_bytes())763                }764            }765            // FIXME: `min_align_of_val` was renamed to `align_of_val` in Rust 1.89766            // (https://github.com/rust-lang/rust/pull/142410)767            "min_align_of_val" | "align_of_val" => {768                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {769                    return Err(MirEvalError::InternalError(770                        "align_of_val generic arg is not provided".into(),771                    ));772                };773                let [arg] = args else {774                    return Err(MirEvalError::InternalError(775                        "align_of_val args are not provided".into(),776                    ));777                };778                if let Some((_, align)) = self.size_align_of(ty, locals)? {779                    destination.write_from_bytes(self, &align.to_le_bytes())780                } else {781                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);782                    let (_, align) = self.size_align_of_unsized(ty, metadata, locals)?;783                    destination.write_from_bytes(self, &align.to_le_bytes())784                }785            }786            "type_name" => {787                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {788                    return Err(MirEvalError::InternalError(789                        "type_name generic arg is not provided".into(),790                    ));791                };792                let ty_name = match ty.display_source_code(793                    self.db,794                    locals.body.owner.module(self.db),795                    true,796                ) {797                    Ok(ty_name) => ty_name,798                    // Fallback to human readable display in case of `Err`. Ideally we want to use `display_source_code` to799                    // render full paths.800                    Err(_) => {801                        let krate = locals.body.owner.krate(self.db);802                        ty.display(self.db, DisplayTarget::from_crate(self.db, krate)).to_string()803                    }804                };805                let len = ty_name.len();806                let addr = self.heap_allocate(len, 1)?;807                self.write_memory(addr, ty_name.as_bytes())?;808                destination.slice(0..self.ptr_size()).write_from_bytes(self, &addr.to_bytes())?;809                destination810                    .slice(self.ptr_size()..2 * self.ptr_size())811                    .write_from_bytes(self, &len.to_le_bytes())812            }813            "needs_drop" => {814                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {815                    return Err(MirEvalError::InternalError(816                        "size_of generic arg is not provided".into(),817                    ));818                };819                let result = match has_drop_glue(&self.infcx, ty, self.param_env.param_env) {820                    DropGlue::HasDropGlue => true,821                    DropGlue::None => false,822                    DropGlue::DependOnParams => {823                        never!("should be fully monomorphized now");824                        true825                    }826                };827                destination.write_from_bytes(self, &[u8::from(result)])828            }829            "ptr_guaranteed_cmp" => {830                // FIXME: this is wrong for const eval, it should return 2 in some831                // cases.832                let [lhs, rhs] = args else {833                    return Err(MirEvalError::InternalError(834                        "ptr_guaranteed_cmp args are not provided".into(),835                    ));836                };837                let ans = lhs.get(self)? == rhs.get(self)?;838                destination.write_from_bytes(self, &[u8::from(ans)])839            }840            "saturating_add" | "saturating_sub" => {841                let [lhs, rhs] = args else {842                    return Err(MirEvalError::InternalError(843                        "saturating_add args are not provided".into(),844                    ));845                };846                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));847                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));848                let ans = match name {849                    "saturating_add" => lhs.saturating_add(rhs),850                    "saturating_sub" => lhs.saturating_sub(rhs),851                    _ => unreachable!(),852                };853                let bits = destination.size * 8;854                // FIXME: signed855                let is_signed = false;856                let mx: u128 = if is_signed { (1 << (bits - 1)) - 1 } else { (1 << bits) - 1 };857                // FIXME: signed858                let mn: u128 = 0;859                let ans = cmp::min(mx, cmp::max(mn, ans));860                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])861            }862            "wrapping_add" | "unchecked_add" => {863                let [lhs, rhs] = args else {864                    return Err(MirEvalError::InternalError(865                        "wrapping_add args are not provided".into(),866                    ));867                };868                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));869                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));870                let ans = lhs.wrapping_add(rhs);871                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])872            }873            "ptr_offset_from_unsigned" | "ptr_offset_from" => {874                let [lhs, rhs] = args else {875                    return Err(MirEvalError::InternalError(876                        "wrapping_sub args are not provided".into(),877                    ));878                };879                let lhs = i128::from_le_bytes(pad16(lhs.get(self)?, false));880                let rhs = i128::from_le_bytes(pad16(rhs.get(self)?, false));881                let ans = lhs.wrapping_sub(rhs);882                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {883                    return Err(MirEvalError::InternalError(884                        "ptr_offset_from generic arg is not provided".into(),885                    ));886                };887                let size = self.size_of_sized(ty, locals, "ptr_offset_from arg")? as i128;888                let ans = ans / size;889                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])890            }891            "wrapping_sub" | "unchecked_sub" => {892                let [lhs, rhs] = args else {893                    return Err(MirEvalError::InternalError(894                        "wrapping_sub args are not provided".into(),895                    ));896                };897                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));898                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));899                let ans = lhs.wrapping_sub(rhs);900                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])901            }902            "wrapping_mul" | "unchecked_mul" => {903                let [lhs, rhs] = args else {904                    return Err(MirEvalError::InternalError(905                        "wrapping_mul args are not provided".into(),906                    ));907                };908                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));909                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));910                let ans = lhs.wrapping_mul(rhs);911                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])912            }913            "wrapping_shl" | "unchecked_shl" => {914                // FIXME: signed915                let [lhs, rhs] = args else {916                    return Err(MirEvalError::InternalError(917                        "unchecked_shl args are not provided".into(),918                    ));919                };920                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));921                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));922                let ans = lhs.wrapping_shl(rhs as u32);923                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])924            }925            "wrapping_shr" | "unchecked_shr" => {926                // FIXME: signed927                let [lhs, rhs] = args else {928                    return Err(MirEvalError::InternalError(929                        "unchecked_shr args are not provided".into(),930                    ));931                };932                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));933                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));934                let ans = lhs.wrapping_shr(rhs as u32);935                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])936            }937            "unchecked_rem" => {938                // FIXME: signed939                let [lhs, rhs] = args else {940                    return Err(MirEvalError::InternalError(941                        "unchecked_rem args are not provided".into(),942                    ));943                };944                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));945                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));946                let ans = lhs.checked_rem(rhs).ok_or_else(|| {947                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())948                })?;949                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])950            }951            "unchecked_div" | "exact_div" => {952                // FIXME: signed953                let [lhs, rhs] = args else {954                    return Err(MirEvalError::InternalError(955                        "unchecked_div args are not provided".into(),956                    ));957                };958                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));959                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));960                let ans = lhs.checked_div(rhs).ok_or_else(|| {961                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())962                })?;963                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])964            }965            "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => {966                let [lhs, rhs] = args else {967                    return Err(MirEvalError::InternalError(968                        "const_eval_select args are not provided".into(),969                    ));970                };971                let result_ty = Ty::new_tup_from_iter(972                    self.interner(),973                    [lhs.ty, Ty::new_bool(self.interner())].into_iter(),974                );975                let op_size = self.size_of_sized(lhs.ty, locals, "operand of add_with_overflow")?;976                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));977                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));978                let (ans, u128overflow) = match name {979                    "add_with_overflow" => lhs.overflowing_add(rhs),980                    "sub_with_overflow" => lhs.overflowing_sub(rhs),981                    "mul_with_overflow" => lhs.overflowing_mul(rhs),982                    _ => unreachable!(),983                };984                let is_overflow = u128overflow985                    || ans.to_le_bytes()[op_size..].iter().any(|&it| it != 0 && it != 255);986                let is_overflow = vec![u8::from(is_overflow)];987                let layout = self.layout(result_ty)?;988                let result = self.construct_with_layout(989                    layout.size.bytes_usize(),990                    &layout,991                    None,992                    [ans.to_le_bytes()[0..op_size].to_vec(), is_overflow]993                        .into_iter()994                        .map(IntervalOrOwned::Owned),995                )?;996                destination.write_from_bytes(self, &result)997            }998            "copy" | "copy_nonoverlapping" => {999                let [src, dst, offset] = args else {1000                    return Err(MirEvalError::InternalError(1001                        "copy_nonoverlapping args are not provided".into(),1002                    ));1003                };1004                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1005                    return Err(MirEvalError::InternalError(1006                        "copy_nonoverlapping generic arg is not provided".into(),1007                    ));1008                };1009                let src = Address::from_bytes(src.get(self)?)?;1010                let dst = Address::from_bytes(dst.get(self)?)?;1011                let offset = from_bytes!(usize, offset.get(self)?);1012                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;1013                let size = offset * size;1014                let src = Interval { addr: src, size };1015                let dst = Interval { addr: dst, size };1016                dst.write_from_interval(self, src)1017            }1018            "offset" | "arith_offset" => {1019                let [ptr, offset] = args else {1020                    return Err(MirEvalError::InternalError("offset args are not provided".into()));1021                };1022                let ty = if name == "offset" {1023                    let Some(ty0) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1024                        return Err(MirEvalError::InternalError(1025                            "offset generic arg is not provided".into(),1026                        ));1027                    };1028                    let Some(ty1) = generic_args.as_slice().get(1).and_then(|it| it.ty()) else {1029                        return Err(MirEvalError::InternalError(1030                            "offset generic arg is not provided".into(),1031                        ));1032                    };1033                    if !matches!(1034                        ty1.kind(),1035                        TyKind::Int(rustc_type_ir::IntTy::Isize)1036                            | TyKind::Uint(rustc_type_ir::UintTy::Usize)1037                    ) {1038                        return Err(MirEvalError::InternalError(1039                            "offset generic arg is not usize or isize".into(),1040                        ));1041                    }1042                    match ty0.kind() {1043                        TyKind::RawPtr(ty, _) => ty,1044                        _ => {1045                            return Err(MirEvalError::InternalError(1046                                "offset generic arg is not a raw pointer".into(),1047                            ));1048                        }1049                    }1050                } else {1051                    let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1052                        return Err(MirEvalError::InternalError(1053                            "arith_offset generic arg is not provided".into(),1054                        ));1055                    };1056                    ty1057                };1058                let ptr = u128::from_le_bytes(pad16(ptr.get(self)?, false));1059                let offset = u128::from_le_bytes(pad16(offset.get(self)?, false));1060                let size = self.size_of_sized(ty, locals, "offset ptr type")? as u128;1061                let ans = ptr + offset * size;1062                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])1063            }1064            "assert_inhabited" | "assert_zero_valid" | "assert_uninit_valid" | "assume" => {1065                // FIXME: We should actually implement these checks1066                Ok(())1067            }1068            "forget" => {1069                // We don't call any drop glue yet, so there is nothing here1070                Ok(())1071            }1072            "transmute" | "transmute_unchecked" => {1073                let [arg] = args else {1074                    return Err(MirEvalError::InternalError(1075                        "transmute arg is not provided".into(),1076                    ));1077                };1078                destination.write_from_interval(self, arg.interval)1079            }1080            "ctpop" => {1081                let [arg] = args else {1082                    return Err(MirEvalError::InternalError("ctpop arg is not provided".into()));1083                };1084                let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).count_ones();1085                destination1086                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])1087            }1088            "ctlz" | "ctlz_nonzero" => {1089                let [arg] = args else {1090                    return Err(MirEvalError::InternalError("ctlz arg is not provided".into()));1091                };1092                let result =1093                    u128::from_le_bytes(pad16(arg.get(self)?, false)).leading_zeros() as usize;1094                let result = result - (128 - arg.interval.size * 8);1095                destination1096                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])1097            }1098            "cttz" | "cttz_nonzero" => {1099                let [arg] = args else {1100                    return Err(MirEvalError::InternalError("cttz arg is not provided".into()));1101                };1102                let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).trailing_zeros();1103                destination1104                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])1105            }1106            "rotate_left" => {1107                let [lhs, rhs] = args else {1108                    return Err(MirEvalError::InternalError(1109                        "rotate_left args are not provided".into(),1110                    ));1111                };1112                let lhs = &lhs.get(self)?[0..destination.size];1113                let rhs = rhs.get(self)?[0] as u32;1114                match destination.size {1115                    1 => {1116                        let r = from_bytes!(u8, lhs).rotate_left(rhs);1117                        destination.write_from_bytes(self, &r.to_le_bytes())1118                    }1119                    2 => {1120                        let r = from_bytes!(u16, lhs).rotate_left(rhs);1121                        destination.write_from_bytes(self, &r.to_le_bytes())1122                    }1123                    4 => {1124                        let r = from_bytes!(u32, lhs).rotate_left(rhs);1125                        destination.write_from_bytes(self, &r.to_le_bytes())1126                    }1127                    8 => {1128                        let r = from_bytes!(u64, lhs).rotate_left(rhs);1129                        destination.write_from_bytes(self, &r.to_le_bytes())1130                    }1131                    16 => {1132                        let r = from_bytes!(u128, lhs).rotate_left(rhs);1133                        destination.write_from_bytes(self, &r.to_le_bytes())1134                    }1135                    s => not_supported!("destination with size {s} for rotate_left"),1136                }1137            }1138            "rotate_right" => {1139                let [lhs, rhs] = args else {1140                    return Err(MirEvalError::InternalError(1141                        "rotate_right args are not provided".into(),1142                    ));1143                };1144                let lhs = &lhs.get(self)?[0..destination.size];1145                let rhs = rhs.get(self)?[0] as u32;1146                match destination.size {1147                    1 => {1148                        let r = from_bytes!(u8, lhs).rotate_right(rhs);1149                        destination.write_from_bytes(self, &r.to_le_bytes())1150                    }1151                    2 => {1152                        let r = from_bytes!(u16, lhs).rotate_right(rhs);1153                        destination.write_from_bytes(self, &r.to_le_bytes())1154                    }1155                    4 => {1156                        let r = from_bytes!(u32, lhs).rotate_right(rhs);1157                        destination.write_from_bytes(self, &r.to_le_bytes())1158                    }1159                    8 => {1160                        let r = from_bytes!(u64, lhs).rotate_right(rhs);1161                        destination.write_from_bytes(self, &r.to_le_bytes())1162                    }1163                    16 => {1164                        let r = from_bytes!(u128, lhs).rotate_right(rhs);1165                        destination.write_from_bytes(self, &r.to_le_bytes())1166                    }1167                    s => not_supported!("destination with size {s} for rotate_right"),1168                }1169            }1170            "discriminant_value" => {1171                let [arg] = args else {1172                    return Err(MirEvalError::InternalError(1173                        "discriminant_value arg is not provided".into(),1174                    ));1175                };1176                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1177                    return Err(MirEvalError::InternalError(1178                        "discriminant_value generic arg is not provided".into(),1179                    ));1180                };1181                let addr = Address::from_bytes(arg.get(self)?)?;1182                let size = self.size_of_sized(ty, locals, "discriminant_value ptr type")?;1183                let interval = Interval { addr, size };1184                let r = self.compute_discriminant(ty, interval.get(self)?)?;1185                destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])1186            }1187            "const_eval_select" => {1188                let [tuple, const_fn, _] = args else {1189                    return Err(MirEvalError::InternalError(1190                        "const_eval_select args are not provided".into(),1191                    ));1192                };1193                let mut args = vec![const_fn.clone()];1194                let TyKind::Tuple(fields) = tuple.ty.kind() else {1195                    return Err(MirEvalError::InternalError(1196                        "const_eval_select arg[0] is not a tuple".into(),1197                    ));1198                };1199                let layout = self.layout(tuple.ty)?;1200                for (i, field) in fields.iter().enumerate() {1201                    let offset = layout.fields.offset(i).bytes_usize();1202                    let addr = tuple.interval.addr.offset(offset);1203                    args.push(IntervalAndTy::new(addr, field, self, locals)?);1204                }1205                if let Some(target) = self.lang_items().FnOnce1206                    && let Some(def) = target1207                        .trait_items(self.db)1208                        .method_by_name(&Name::new_symbol_root(sym::call_once))1209                {1210                    self.exec_fn_trait(1211                        def,1212                        &args,1213                        // FIXME: wrong for manual impls of `FnOnce`1214                        GenericArgs::empty(self.interner()),1215                        locals,1216                        destination,1217                        None,1218                        span,1219                    )?;1220                    return Ok(true);1221                }1222                not_supported!("FnOnce was not available for executing const_eval_select");1223            }1224            "read_via_copy" | "volatile_load" => {1225                let [arg] = args else {1226                    return Err(MirEvalError::InternalError(1227                        "read_via_copy args are not provided".into(),1228                    ));1229                };1230                let addr = Address::from_bytes(arg.interval.get(self)?)?;1231                destination.write_from_interval(self, Interval { addr, size: destination.size })1232            }1233            "write_via_move" => {1234                let [ptr, val] = args else {1235                    return Err(MirEvalError::InternalError(1236                        "write_via_move args are not provided".into(),1237                    ));1238                };1239                let dst = Address::from_bytes(ptr.get(self)?)?;1240                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1241                    return Err(MirEvalError::InternalError(1242                        "write_via_copy generic arg is not provided".into(),1243                    ));1244                };1245                let size = self.size_of_sized(ty, locals, "write_via_move ptr type")?;1246                Interval { addr: dst, size }.write_from_interval(self, val.interval)?;1247                Ok(())1248            }1249            "write_bytes" => {1250                let [dst, val, count] = args else {1251                    return Err(MirEvalError::InternalError(1252                        "write_bytes args are not provided".into(),1253                    ));1254                };1255                let count = from_bytes!(usize, count.get(self)?);1256                let val = from_bytes!(u8, val.get(self)?);1257                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1258                    return Err(MirEvalError::InternalError(1259                        "write_bytes generic arg is not provided".into(),1260                    ));1261                };1262                let dst = Address::from_bytes(dst.get(self)?)?;1263                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;1264                let size = count * size;1265                self.write_memory_using_ref(dst, size)?.fill(val);1266                Ok(())1267            }1268            "ptr_metadata" => {1269                let [ptr] = args else {1270                    return Err(MirEvalError::InternalError(1271                        "ptr_metadata args are not provided".into(),1272                    ));1273                };1274                let arg = ptr.interval.get(self)?.to_owned();1275                let metadata = &arg[self.ptr_size()..];1276                destination.write_from_bytes(self, metadata)?;1277                Ok(())1278            }1279            "three_way_compare" => {1280                let [lhs, rhs] = args else {1281                    return Err(MirEvalError::InternalError(1282                        "three_way_compare args are not provided".into(),1283                    ));1284                };1285                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1286                    return Err(MirEvalError::InternalError(1287                        "three_way_compare generic arg is not provided".into(),1288                    ));1289                };1290                let signed = match ty.kind() {1291                    TyKind::Int(_) => true,1292                    TyKind::Uint(_) => false,1293                    _ => {1294                        return Err(MirEvalError::InternalError(1295                            "three_way_compare expects an integral type".into(),1296                        ));1297                    }1298                };1299                let rhs = rhs.get(self)?;1300                let lhs = lhs.get(self)?;1301                let mut result = Ordering::Equal;1302                for (l, r) in lhs.iter().zip(rhs).rev() {1303                    let it = l.cmp(r);1304                    if it != Ordering::Equal {1305                        result = it;1306                        break;1307                    }1308                }1309                if signed1310                    && let Some((&l, &r)) = lhs.iter().zip(rhs).next_back()1311                    && l != r1312                {1313                    result = (l as i8).cmp(&(r as i8));1314                }1315                if let Some(e) = self.lang_items().Ordering {1316                    let ty = self.db.ty(e.into()).skip_binder();1317                    let r = self.compute_discriminant(ty, &[result as i8 as u8])?;1318                    destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])?;1319                    Ok(())1320                } else {1321                    Err(MirEvalError::InternalError("Ordering enum not found".into()))1322                }1323            }1324            "aggregate_raw_ptr" => {1325                let [data, meta] = args else {1326                    return Err(MirEvalError::InternalError(1327                        "aggregate_raw_ptr args are not provided".into(),1328                    ));1329                };1330                destination.write_from_interval(self, data.interval)?;1331                Interval {1332                    addr: destination.addr.offset(data.interval.size),1333                    size: destination.size - data.interval.size,1334                }1335                .write_from_interval(self, meta.interval)?;1336                Ok(())1337            }1338            _ if needs_override => not_supported!("intrinsic {name} is not implemented"),1339            _ => return Ok(false),1340        }1341        .map(|()| true)1342    }13431344    fn size_align_of_unsized(1345        &mut self,1346        ty: Ty<'db>,1347        metadata: Interval,1348        locals: &Locals,1349    ) -> Result<'db, (usize, usize)> {1350        Ok(match ty.kind() {1351            TyKind::Str => (from_bytes!(usize, metadata.get(self)?), 1),1352            TyKind::Slice(inner) => {1353                let len = from_bytes!(usize, metadata.get(self)?);1354                let (size, align) = self.size_align_of_sized(inner, locals, "slice inner type")?;1355                (size * len, align)1356            }1357            TyKind::Dynamic(..) => self.size_align_of_sized(1358                self.vtable_map.ty_of_bytes(metadata.get(self)?)?,1359                locals,1360                "dyn concrete type",1361            )?,1362            TyKind::Adt(adt_def, subst) => {1363                let id = adt_def.def_id().0;1364                let layout = self.layout_adt(id, subst)?;1365                let id = match id {1366                    AdtId::StructId(s) => s,1367                    _ => not_supported!("unsized enum or union"),1368                };1369                let field_types = self.db.field_types(id.into());1370                let last_field_ty = field_types1371                    .iter()1372                    .next_back()1373                    .unwrap()1374                    .11375                    .get()1376                    .instantiate(self.interner(), subst);1377                let sized_part_size =1378                    layout.fields.offset(field_types.iter().count() - 1).bytes_usize();1379                let sized_part_align = layout.align.bytes() as usize;1380                let (unsized_part_size, unsized_part_align) =1381                    self.size_align_of_unsized(last_field_ty, metadata, locals)?;1382                let align = sized_part_align.max(unsized_part_align) as isize;1383                let size = (sized_part_size + unsized_part_size) as isize;1384                // Must add any necessary padding to `size`1385                // (to make it a multiple of `align`) before returning it.1386                //1387                // Namely, the returned size should be, in C notation:1388                //1389                //   `size + ((size & (align-1)) ? align : 0)`1390                //1391                // emulated via the semi-standard fast bit trick:1392                //1393                //   `(size + (align-1)) & -align`1394                let size = (size + (align - 1)) & (-align);1395                (size as usize, align as usize)1396            }1397            _ => not_supported!("unsized type other than str, slice, struct and dyn"),1398        })1399    }14001401    fn exec_atomic_intrinsic(1402        &mut self,1403        name: &str,1404        args: &[IntervalAndTy<'db>],1405        generic_args: GenericArgs<'db>,1406        destination: Interval,1407        locals: &Locals,1408        _span: MirSpan,1409    ) -> Result<'db, ()> {1410        // We are a single threaded runtime with no UB checking and no optimization, so1411        // we can implement atomic intrinsics as normal functions.14121413        if name.starts_with("singlethreadfence_") || name.starts_with("fence_") {1414            return Ok(());1415        }14161417        // The rest of atomic intrinsics have exactly one generic arg14181419        let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1420            return Err(MirEvalError::InternalError(1421                "atomic intrinsic generic arg is not provided".into(),1422            ));1423        };1424        let Some(arg0) = args.first() else {1425            return Err(MirEvalError::InternalError(1426                "atomic intrinsic arg0 is not provided".into(),1427            ));1428        };1429        let arg0_addr = Address::from_bytes(arg0.get(self)?)?;1430        let arg0_interval =1431            Interval::new(arg0_addr, self.size_of_sized(ty, locals, "atomic intrinsic type arg")?);1432        if name.starts_with("load_") {1433            return destination.write_from_interval(self, arg0_interval);1434        }1435        let Some(arg1) = args.get(1) else {1436            return Err(MirEvalError::InternalError(1437                "atomic intrinsic arg1 is not provided".into(),1438            ));1439        };1440        if name.starts_with("store_") {1441            return arg0_interval.write_from_interval(self, arg1.interval);1442        }1443        if name.starts_with("xchg_") {1444            destination.write_from_interval(self, arg0_interval)?;1445            return arg0_interval.write_from_interval(self, arg1.interval);1446        }1447        if name.starts_with("xadd_") {1448            destination.write_from_interval(self, arg0_interval)?;1449            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1450            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1451            let ans = lhs.wrapping_add(rhs);1452            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1453        }1454        if name.starts_with("xsub_") {1455            destination.write_from_interval(self, arg0_interval)?;1456            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1457            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1458            let ans = lhs.wrapping_sub(rhs);1459            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1460        }1461        if name.starts_with("and_") {1462            destination.write_from_interval(self, arg0_interval)?;1463            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1464            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1465            let ans = lhs & rhs;1466            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1467        }1468        if name.starts_with("or_") {1469            destination.write_from_interval(self, arg0_interval)?;1470            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1471            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1472            let ans = lhs | rhs;1473            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1474        }1475        if name.starts_with("xor_") {1476            destination.write_from_interval(self, arg0_interval)?;1477            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1478            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1479            let ans = lhs ^ rhs;1480            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1481        }1482        if name.starts_with("nand_") {1483            destination.write_from_interval(self, arg0_interval)?;1484            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1485            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1486            let ans = !(lhs & rhs);1487            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1488        }1489        let Some(arg2) = args.get(2) else {1490            return Err(MirEvalError::InternalError(1491                "atomic intrinsic arg2 is not provided".into(),1492            ));1493        };1494        if name.starts_with("cxchg_") || name.starts_with("cxchgweak_") {1495            let dest = if arg1.get(self)? == arg0_interval.get(self)? {1496                arg0_interval.write_from_interval(self, arg2.interval)?;1497                (arg1.interval, true)1498            } else {1499                (arg0_interval, false)1500            };1501            let result_ty = Ty::new_tup_from_iter(1502                self.interner(),1503                [ty, Ty::new_bool(self.interner())].into_iter(),1504            );1505            let layout = self.layout(result_ty)?;1506            let result = self.construct_with_layout(1507                layout.size.bytes_usize(),1508                &layout,1509                None,1510                [IntervalOrOwned::Borrowed(dest.0), IntervalOrOwned::Owned(vec![u8::from(dest.1)])]1511                    .into_iter(),1512            )?;1513            return destination.write_from_bytes(self, &result);1514        }1515        not_supported!("unknown atomic intrinsic {name}");1516    }1517}