1//! Interpret intrinsics, lang items and `extern "C"` wellknown functions which their implementation2//! is not available.3//!4use std::cmp::{self, Ordering};56use hir_def::{attrs::AttrFlags, signatures::FunctionSignature};7use rustc_abi::ExternAbi;8use rustc_type_ir::inherent::{GenericArgs as _, IntoKind, SliceLike, Ty as _};9use stdx::never;1011use crate::{12    display::DisplayTarget,13    drop::{DropGlue, has_drop_glue},14    mir::eval::{15        Address, AdtId, Arc, Evaluator, FunctionId, GenericArgs, HasModule, HirDisplay, Interval,16        IntervalAndTy, IntervalOrOwned, ItemContainerId, Layout, Locals, Lookup, MirEvalError,17        MirSpan, Mutability, Result, Ty, TyKind, from_bytes, not_supported, pad16,18    },19    next_solver::Region,20};2122mod simd;2324#[derive(Debug, Clone, Copy, PartialEq, Eq)]25enum EvalLangItem {26    BeginPanic,27    SliceLen,28    DropInPlace,29}3031impl<'a, 'db: 'a> Evaluator<'a, 'db> {32    pub(super) fn detect_and_exec_special_function(33        &mut self,34        def: FunctionId,35        args: &[IntervalAndTy<'db>],36        generic_args: GenericArgs<'db>,37        locals: &Locals<'a>,38        destination: Interval,39        span: MirSpan,40    ) -> Result<'db, bool> {41        if self.not_special_fn_cache.borrow().contains(&def) {42            return Ok(false);43        }4445        let function_data = FunctionSignature::of(self.db, def);46        let attrs = AttrFlags::query(self.db, def.into());47        let is_intrinsic = FunctionSignature::is_intrinsic(self.db, def);4849        if is_intrinsic {50            return self.exec_intrinsic(51                function_data.name.as_str(),52                args,53                generic_args,54                destination,55                locals,56                span,57                !function_data.has_body()58                    || attrs.contains(AttrFlags::RUSTC_INTRINSIC_MUST_BE_OVERRIDDEN),59            );60        }61        let is_extern_c = match def.lookup(self.db).container {62            hir_def::ItemContainerId::ExternBlockId(block) => {63                matches!(block.abi(self.db), ExternAbi::C { .. })64            }65            _ => false,66        };67        if is_extern_c {68            return self69                .exec_extern_c(70                    function_data.name.as_str(),71                    args,72                    generic_args,73                    destination,74                    locals,75                    span,76                )77                .map(|()| true);78        }7980        if attrs.intersects(81            AttrFlags::RUSTC_ALLOCATOR82                | AttrFlags::RUSTC_DEALLOCATOR83                | AttrFlags::RUSTC_REALLOCATOR84                | AttrFlags::RUSTC_ALLOCATOR_ZEROED,85        ) {86            self.exec_alloc_fn(attrs, args, destination)?;87            return Ok(true);88        }89        if let Some(it) = self.detect_lang_function(def) {90            let result = self.exec_lang_item(it, generic_args, args, locals, span)?;91            destination.write_from_bytes(self, &result)?;92            return Ok(true);93        }94        if let ItemContainerId::TraitId(t) = def.lookup(self.db).container95            && Some(t) == self.lang_items().Clone96        {97            let [self_ty] = generic_args.as_slice() else {98                not_supported!("wrong generic arg count for clone");99            };100            let Some(self_ty) = self_ty.ty() else {101                not_supported!("wrong generic arg kind for clone");102            };103            // Clone has special impls for tuples and function pointers104            if matches!(self_ty.kind(), TyKind::FnPtr(..) | TyKind::Tuple(..) | TyKind::Closure(..))105            {106                self.exec_clone(def, args, self_ty, locals, destination, span)?;107                return Ok(true);108            }109            // Return early to prevent caching clone as non special fn.110            return Ok(false);111        }112        self.not_special_fn_cache.borrow_mut().insert(def);113        Ok(false)114    }115116    pub(super) fn detect_and_redirect_special_function(117        &mut self,118        def: FunctionId,119    ) -> Result<'db, Option<FunctionId>> {120        // `PanicFmt` is redirected to `ConstPanicFmt`121        if Some(def) == self.lang_items().PanicFmt {122            let Some(const_panic_fmt) = self.lang_items().ConstPanicFmt else {123                not_supported!("const_panic_fmt lang item not found or not a function");124            };125            return Ok(Some(const_panic_fmt));126        }127        Ok(None)128    }129130    /// Clone has special impls for tuples and function pointers131    fn exec_clone(132        &mut self,133        def: FunctionId,134        args: &[IntervalAndTy<'db>],135        self_ty: Ty<'db>,136        locals: &Locals<'a>,137        destination: Interval,138        span: MirSpan,139    ) -> Result<'db, ()> {140        match self_ty.kind() {141            TyKind::FnPtr(..) => {142                let [arg] = args else {143                    not_supported!("wrong arg count for clone");144                };145                let addr = Address::from_bytes(arg.get(self)?)?;146                return destination147                    .write_from_interval(self, Interval { addr, size: destination.size });148            }149            TyKind::Closure(_, closure_args) => self.exec_clone(150                def,151                args,152                closure_args.as_closure().tupled_upvars_ty(),153                locals,154                destination,155                span,156            )?,157            TyKind::Tuple(subst) => {158                let [arg] = args else {159                    not_supported!("wrong arg count for clone");160                };161                let addr = Address::from_bytes(arg.get(self)?)?;162                let layout = self.layout(self_ty)?;163                self.exec_clone_for_fields(164                    subst.iter(),165                    layout,166                    addr,167                    def,168                    locals,169                    destination,170                    span,171                )?;172            }173            _ => {174                self.exec_fn_with_args(175                    def,176                    args,177                    GenericArgs::new_from_slice(&[self_ty.into()]),178                    locals,179                    destination,180                    None,181                    span,182                )?;183            }184        }185        Ok(())186    }187188    fn exec_clone_for_fields(189        &mut self,190        ty_iter: impl Iterator<Item = Ty<'db>>,191        layout: Arc<Layout>,192        addr: Address,193        def: FunctionId,194        locals: &Locals<'a>,195        destination: Interval,196        span: MirSpan,197    ) -> Result<'db, ()> {198        for (i, ty) in ty_iter.enumerate() {199            let size = self.layout(ty)?.size.bytes_usize();200            let tmp = self.heap_allocate(self.ptr_size(), self.ptr_size())?;201            let arg = IntervalAndTy {202                interval: Interval { addr: tmp, size: self.ptr_size() },203                ty: Ty::new_ref(204                    self.interner(),205                    Region::error(self.interner()),206                    ty,207                    Mutability::Not,208                ),209            };210            let offset = layout.fields.offset(i).bytes_usize();211            self.write_memory(tmp, &addr.offset(offset).to_bytes())?;212            self.exec_clone(213                def,214                &[arg],215                ty,216                locals,217                destination.slice(offset..offset + size),218                span,219            )?;220        }221        Ok(())222    }223224    fn exec_alloc_fn(225        &mut self,226        alloc_fn: AttrFlags,227        args: &[IntervalAndTy<'db>],228        destination: Interval,229    ) -> Result<'db, ()> {230        match alloc_fn {231            _ if alloc_fn232                .intersects(AttrFlags::RUSTC_ALLOCATOR_ZEROED | AttrFlags::RUSTC_ALLOCATOR) =>233            {234                let [size, align] = args else {235                    return Err(MirEvalError::InternalError(236                        "rustc_allocator args are not provided".into(),237                    ));238                };239                let size = from_bytes!(usize, size.get(self)?);240                let align = from_bytes!(usize, align.get(self)?);241                let result = self.heap_allocate(size, align)?;242                destination.write_from_bytes(self, &result.to_bytes())?;243            }244            _ if alloc_fn.contains(AttrFlags::RUSTC_DEALLOCATOR) => { /* no-op for now */ }245            _ if alloc_fn.contains(AttrFlags::RUSTC_REALLOCATOR) => {246                let [ptr, old_size, align, new_size] = args else {247                    return Err(MirEvalError::InternalError(248                        "rustc_allocator args are not provided".into(),249                    ));250                };251                let old_size = from_bytes!(usize, old_size.get(self)?);252                let new_size = from_bytes!(usize, new_size.get(self)?);253                if old_size >= new_size {254                    destination.write_from_interval(self, ptr.interval)?;255                } else {256                    let ptr = Address::from_bytes(ptr.get(self)?)?;257                    let align = from_bytes!(usize, align.get(self)?);258                    let result = self.heap_allocate(new_size, align)?;259                    Interval { addr: result, size: old_size }260                        .write_from_interval(self, Interval { addr: ptr, size: old_size })?;261                    destination.write_from_bytes(self, &result.to_bytes())?;262                }263            }264            _ => not_supported!("unknown alloc function"),265        }266        Ok(())267    }268269    fn detect_lang_function(&self, def: FunctionId) -> Option<EvalLangItem> {270        use EvalLangItem::*;271        let lang_items = self.lang_items();272        let attrs = AttrFlags::query(self.db, def.into());273274        if attrs.contains(AttrFlags::RUSTC_CONST_PANIC_STR) {275            // `#[rustc_const_panic_str]` is treated like `lang = "begin_panic"` by rustc CTFE.276            return Some(BeginPanic);277        }278279        // We want to execute these functions with special logic280        // `PanicFmt` is not detected here as it's redirected later.281        if let Some((_, candidate)) = [282            (lang_items.BeginPanic, BeginPanic),283            (lang_items.SliceLen, SliceLen),284            (lang_items.DropInPlace, DropInPlace),285        ]286        .iter()287        .find(|&(candidate, _)| candidate == Some(def))288        {289            return Some(candidate);290        }291292        None293    }294295    fn exec_lang_item(296        &mut self,297        it: EvalLangItem,298        generic_args: GenericArgs<'db>,299        args: &[IntervalAndTy<'db>],300        locals: &Locals<'a>,301        span: MirSpan,302    ) -> Result<'db, Vec<u8>> {303        use EvalLangItem::*;304        let mut args = args.iter();305        match it {306            BeginPanic => {307                let mut arg = args308                    .next()309                    .ok_or(MirEvalError::InternalError(310                        "argument of BeginPanic is not provided".into(),311                    ))?312                    .clone();313                while let TyKind::Ref(_, ty, _) = arg.ty.kind() {314                    if ty.is_str() {315                        let (pointee, metadata) = arg.interval.get(self)?.split_at(self.ptr_size());316                        let len = from_bytes!(usize, metadata);317318                        return {319                            Err(MirEvalError::Panic(320                                std::str::from_utf8(321                                    self.read_memory(Address::from_bytes(pointee)?, len)?,322                                )323                                .unwrap()324                                .to_owned(),325                            ))326                        };327                    }328                    let size = self.size_of_sized(ty, locals, "begin panic arg")?;329                    let pointee = arg.interval.get(self)?;330                    arg = IntervalAndTy {331                        interval: Interval::new(Address::from_bytes(pointee)?, size),332                        ty,333                    };334                }335                Err(MirEvalError::Panic(format!("unknown-panic-payload: {:?}", arg.ty.kind())))336            }337            SliceLen => {338                let arg = args.next().ok_or(MirEvalError::InternalError(339                    "argument of <[T]>::len() is not provided".into(),340                ))?;341                let arg = arg.get(self)?;342                let ptr_size = arg.len() / 2;343                Ok(arg[ptr_size..].into())344            }345            DropInPlace => {346                let ty = generic_args.as_slice().first().and_then(|it| it.ty()).ok_or(347                    MirEvalError::InternalError(348                        "generic argument of drop_in_place is not provided".into(),349                    ),350                )?;351                let arg = args.next().ok_or(MirEvalError::InternalError(352                    "argument of drop_in_place is not provided".into(),353                ))?;354                let arg = arg.interval.get(self)?.to_owned();355                self.run_drop_glue_deep(356                    ty,357                    locals,358                    Address::from_bytes(&arg[0..self.ptr_size()])?,359                    &arg[self.ptr_size()..],360                    span,361                )?;362                Ok(vec![])363            }364        }365    }366367    fn exec_syscall(368        &mut self,369        id: i64,370        args: &[IntervalAndTy<'db>],371        destination: Interval,372        _locals: &Locals<'a>,373        _span: MirSpan,374    ) -> Result<'db, ()> {375        match id {376            318 => {377                // SYS_getrandom378                let [buf, len, _flags] = args else {379                    return Err(MirEvalError::InternalError(380                        "SYS_getrandom args are not provided".into(),381                    ));382                };383                let addr = Address::from_bytes(buf.get(self)?)?;384                let size = from_bytes!(usize, len.get(self)?);385                for i in 0..size {386                    let rand_byte = self.random_state.rand_u64() as u8;387                    self.write_memory(addr.offset(i), &[rand_byte])?;388                }389                destination.write_from_interval(self, len.interval)390            }391            _ => {392                not_supported!("Unknown syscall id {id:?}")393            }394        }395    }396397    fn exec_extern_c(398        &mut self,399        as_str: &str,400        args: &[IntervalAndTy<'db>],401        _generic_args: GenericArgs<'db>,402        destination: Interval,403        locals: &Locals<'a>,404        span: MirSpan,405    ) -> Result<'db, ()> {406        match as_str {407            "memcmp" => {408                let [ptr1, ptr2, size] = args else {409                    return Err(MirEvalError::InternalError("memcmp args are not provided".into()));410                };411                let addr1 = Address::from_bytes(ptr1.get(self)?)?;412                let addr2 = Address::from_bytes(ptr2.get(self)?)?;413                let size = from_bytes!(usize, size.get(self)?);414                let slice1 = self.read_memory(addr1, size)?;415                let slice2 = self.read_memory(addr2, size)?;416                let r: i128 = match slice1.cmp(slice2) {417                    cmp::Ordering::Less => -1,418                    cmp::Ordering::Equal => 0,419                    cmp::Ordering::Greater => 1,420                };421                destination.write_from_bytes(self, &r.to_le_bytes()[..destination.size])422            }423            "write" => {424                let [fd, ptr, len] = args else {425                    return Err(MirEvalError::InternalError(426                        "libc::write args are not provided".into(),427                    ));428                };429                let fd = u128::from_le_bytes(pad16(fd.get(self)?, false));430                let interval = Interval {431                    addr: Address::from_bytes(ptr.get(self)?)?,432                    size: from_bytes!(usize, len.get(self)?),433                };434                match fd {435                    1 => {436                        self.write_to_stdout(interval)?;437                    }438                    2 => {439                        self.write_to_stderr(interval)?;440                    }441                    _ => not_supported!("write to arbitrary file descriptor"),442                }443                destination.write_from_interval(self, len.interval)?;444                Ok(())445            }446            "pthread_key_create" => {447                let key = self.thread_local_storage.create_key();448                let Some(arg0) = args.first() else {449                    return Err(MirEvalError::InternalError(450                        "pthread_key_create arg0 is not provided".into(),451                    ));452                };453                let arg0_addr = Address::from_bytes(arg0.get(self)?)?;454                let key_ty = if let Some((ty, ..)) = arg0.ty.as_reference_or_ptr() {455                    ty456                } else {457                    return Err(MirEvalError::InternalError(458                        "pthread_key_create arg0 is not a pointer".into(),459                    ));460                };461                let arg0_interval = Interval::new(462                    arg0_addr,463                    self.size_of_sized(key_ty, locals, "pthread_key_create key arg")?,464                );465                arg0_interval.write_from_bytes(self, &key.to_le_bytes()[0..arg0_interval.size])?;466                // return 0 as success467                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;468                Ok(())469            }470            "pthread_getspecific" => {471                let Some(arg0) = args.first() else {472                    return Err(MirEvalError::InternalError(473                        "pthread_getspecific arg0 is not provided".into(),474                    ));475                };476                let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]);477                let value = self.thread_local_storage.get_key(key)?;478                destination.write_from_bytes(self, &value.to_le_bytes()[0..destination.size])?;479                Ok(())480            }481            "pthread_setspecific" => {482                let Some(arg0) = args.first() else {483                    return Err(MirEvalError::InternalError(484                        "pthread_setspecific arg0 is not provided".into(),485                    ));486                };487                let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]);488                let Some(arg1) = args.get(1) else {489                    return Err(MirEvalError::InternalError(490                        "pthread_setspecific arg1 is not provided".into(),491                    ));492                };493                let value = from_bytes!(u128, pad16(arg1.get(self)?, false));494                self.thread_local_storage.set_key(key, value)?;495                // return 0 as success496                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;497                Ok(())498            }499            "pthread_key_delete" => {500                // we ignore this currently501                // return 0 as success502                destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?;503                Ok(())504            }505            "syscall" => {506                let Some((id, rest)) = args.split_first() else {507                    return Err(MirEvalError::InternalError("syscall arg1 is not provided".into()));508                };509                let id = from_bytes!(i64, id.get(self)?);510                self.exec_syscall(id, rest, destination, locals, span)511            }512            "sched_getaffinity" => {513                let [_pid, _set_size, set] = args else {514                    return Err(MirEvalError::InternalError(515                        "sched_getaffinity args are not provided".into(),516                    ));517                };518                let set = Address::from_bytes(set.get(self)?)?;519                // Only enable core 0 (we are single threaded anyway), which is bitset 0x0000001520                self.write_memory(set, &[1])?;521                // return 0 as success522                self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);523                Ok(())524            }525            "getenv" => {526                let [name] = args else {527                    return Err(MirEvalError::InternalError("getenv args are not provided".into()));528                };529                let mut name_buf = vec![];530                let name = {531                    let mut index = Address::from_bytes(name.get(self)?)?;532                    loop {533                        let byte = self.read_memory(index, 1)?[0];534                        index = index.offset(1);535                        if byte == 0 {536                            break;537                        }538                        name_buf.push(byte);539                    }540                    String::from_utf8_lossy(&name_buf)541                };542                let value = self.crate_id.env(self.db).get(&name);543                match value {544                    None => {545                        // Write null as fail546                        self.write_memory_using_ref(destination.addr, destination.size)?.fill(0);547                    }548                    Some(mut value) => {549                        value.push('\0');550                        let addr = self.heap_allocate(value.len(), 1)?;551                        self.write_memory(addr, value.as_bytes())?;552                        self.write_memory(destination.addr, &addr.to_bytes())?;553                    }554                }555                Ok(())556            }557            _ => not_supported!("unknown external function {as_str}"),558        }559    }560561    fn exec_intrinsic(562        &mut self,563        name: &str,564        args: &[IntervalAndTy<'db>],565        generic_args: GenericArgs<'db>,566        destination: Interval,567        locals: &Locals<'a>,568        span: MirSpan,569        needs_override: bool,570    ) -> Result<'db, bool> {571        if let Some(name) = name.strip_prefix("atomic_") {572            return self573                .exec_atomic_intrinsic(name, args, generic_args, destination, locals, span)574                .map(|()| true);575        }576        if let Some(name) = name.strip_prefix("simd_") {577            return self578                .exec_simd_intrinsic(name, args, generic_args, destination, locals, span)579                .map(|()| true);580        }581        // FIXME(#17451): Add `f16` and `f128` intrinsics.582        if let Some(name) = name.strip_suffix("f64") {583            let result = match name {584                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"585                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {586                    let [arg] = args else {587                        return Err(MirEvalError::InternalError(588                            "f64 intrinsic signature doesn't match fn (f64) -> f64".into(),589                        ));590                    };591                    let arg = from_bytes!(f64, arg.get(self)?);592                    match name {593                        "sqrt" => arg.sqrt(),594                        "sin" => arg.sin(),595                        "cos" => arg.cos(),596                        "exp" => arg.exp(),597                        "exp2" => arg.exp2(),598                        "log" => arg.ln(),599                        "log10" => arg.log10(),600                        "log2" => arg.log2(),601                        "fabs" => arg.abs(),602                        "floor" => arg.floor(),603                        "ceil" => arg.ceil(),604                        "trunc" => arg.trunc(),605                        // FIXME: these rounds should be different, but only `.round()` is stable now.606                        "rint" => arg.round(),607                        "nearbyint" => arg.round(),608                        "round" => arg.round(),609                        "roundeven" => arg.round(),610                        _ => unreachable!(),611                    }612                }613                "pow" | "minnum" | "maxnum" | "copysign" => {614                    let [arg1, arg2] = args else {615                        return Err(MirEvalError::InternalError(616                            "f64 intrinsic signature doesn't match fn (f64, f64) -> f64".into(),617                        ));618                    };619                    let arg1 = from_bytes!(f64, arg1.get(self)?);620                    let arg2 = from_bytes!(f64, arg2.get(self)?);621                    match name {622                        "pow" => arg1.powf(arg2),623                        "minnum" => arg1.min(arg2),624                        "maxnum" => arg1.max(arg2),625                        "copysign" => arg1.copysign(arg2),626                        _ => unreachable!(),627                    }628                }629                "powi" => {630                    let [arg1, arg2] = args else {631                        return Err(MirEvalError::InternalError(632                            "powif64 signature doesn't match fn (f64, i32) -> f64".into(),633                        ));634                    };635                    let arg1 = from_bytes!(f64, arg1.get(self)?);636                    let arg2 = from_bytes!(i32, arg2.get(self)?);637                    arg1.powi(arg2)638                }639                "fma" => {640                    let [arg1, arg2, arg3] = args else {641                        return Err(MirEvalError::InternalError(642                            "fmaf64 signature doesn't match fn (f64, f64, f64) -> f64".into(),643                        ));644                    };645                    let arg1 = from_bytes!(f64, arg1.get(self)?);646                    let arg2 = from_bytes!(f64, arg2.get(self)?);647                    let arg3 = from_bytes!(f64, arg3.get(self)?);648                    arg1.mul_add(arg2, arg3)649                }650                _ => not_supported!("unknown f64 intrinsic {name}"),651            };652            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);653        }654        if let Some(name) = name.strip_suffix("f32") {655            let result = match name {656                "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs"657                | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => {658                    let [arg] = args else {659                        return Err(MirEvalError::InternalError(660                            "f32 intrinsic signature doesn't match fn (f32) -> f32".into(),661                        ));662                    };663                    let arg = from_bytes!(f32, arg.get(self)?);664                    match name {665                        "sqrt" => arg.sqrt(),666                        "sin" => arg.sin(),667                        "cos" => arg.cos(),668                        "exp" => arg.exp(),669                        "exp2" => arg.exp2(),670                        "log" => arg.ln(),671                        "log10" => arg.log10(),672                        "log2" => arg.log2(),673                        "fabs" => arg.abs(),674                        "floor" => arg.floor(),675                        "ceil" => arg.ceil(),676                        "trunc" => arg.trunc(),677                        // FIXME: these rounds should be different, but only `.round()` is stable now.678                        "rint" => arg.round(),679                        "nearbyint" => arg.round(),680                        "round" => arg.round(),681                        "roundeven" => arg.round(),682                        _ => unreachable!(),683                    }684                }685                "pow" | "minnum" | "maxnum" | "copysign" => {686                    let [arg1, arg2] = args else {687                        return Err(MirEvalError::InternalError(688                            "f32 intrinsic signature doesn't match fn (f32, f32) -> f32".into(),689                        ));690                    };691                    let arg1 = from_bytes!(f32, arg1.get(self)?);692                    let arg2 = from_bytes!(f32, arg2.get(self)?);693                    match name {694                        "pow" => arg1.powf(arg2),695                        "minnum" => arg1.min(arg2),696                        "maxnum" => arg1.max(arg2),697                        "copysign" => arg1.copysign(arg2),698                        _ => unreachable!(),699                    }700                }701                "powi" => {702                    let [arg1, arg2] = args else {703                        return Err(MirEvalError::InternalError(704                            "powif32 signature doesn't match fn (f32, i32) -> f32".into(),705                        ));706                    };707                    let arg1 = from_bytes!(f32, arg1.get(self)?);708                    let arg2 = from_bytes!(i32, arg2.get(self)?);709                    arg1.powi(arg2)710                }711                "fma" => {712                    let [arg1, arg2, arg3] = args else {713                        return Err(MirEvalError::InternalError(714                            "fmaf32 signature doesn't match fn (f32, f32, f32) -> f32".into(),715                        ));716                    };717                    let arg1 = from_bytes!(f32, arg1.get(self)?);718                    let arg2 = from_bytes!(f32, arg2.get(self)?);719                    let arg3 = from_bytes!(f32, arg3.get(self)?);720                    arg1.mul_add(arg2, arg3)721                }722                _ => not_supported!("unknown f32 intrinsic {name}"),723            };724            return destination.write_from_bytes(self, &result.to_le_bytes()).map(|()| true);725        }726        match name {727            "size_of" => {728                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {729                    return Err(MirEvalError::InternalError(730                        "size_of generic arg is not provided".into(),731                    ));732                };733                let size = self.size_of_sized(ty, locals, "size_of arg")?;734                destination.write_from_bytes(self, &size.to_le_bytes()[0..destination.size])735            }736            // FIXME: `min_align_of` was renamed to `align_of` in Rust 1.89737            // (https://github.com/rust-lang/rust/pull/142410)738            "min_align_of" | "align_of" => {739                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {740                    return Err(MirEvalError::InternalError(741                        "align_of generic arg is not provided".into(),742                    ));743                };744                let align = self.layout(ty)?.align.bytes();745                destination.write_from_bytes(self, &align.to_le_bytes()[0..destination.size])746            }747            "size_of_val" => {748                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {749                    return Err(MirEvalError::InternalError(750                        "size_of_val generic arg is not provided".into(),751                    ));752                };753                let [arg] = args else {754                    return Err(MirEvalError::InternalError(755                        "size_of_val args are not provided".into(),756                    ));757                };758                if let Some((size, _)) = self.size_align_of(ty, locals)? {759                    destination.write_from_bytes(self, &size.to_le_bytes())760                } else {761                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);762                    let (size, _) = self.size_align_of_unsized(ty, metadata, locals)?;763                    destination.write_from_bytes(self, &size.to_le_bytes())764                }765            }766            // FIXME: `min_align_of_val` was renamed to `align_of_val` in Rust 1.89767            // (https://github.com/rust-lang/rust/pull/142410)768            "min_align_of_val" | "align_of_val" => {769                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {770                    return Err(MirEvalError::InternalError(771                        "align_of_val generic arg is not provided".into(),772                    ));773                };774                let [arg] = args else {775                    return Err(MirEvalError::InternalError(776                        "align_of_val args are not provided".into(),777                    ));778                };779                if let Some((_, align)) = self.size_align_of(ty, locals)? {780                    destination.write_from_bytes(self, &align.to_le_bytes())781                } else {782                    let metadata = arg.interval.slice(self.ptr_size()..self.ptr_size() * 2);783                    let (_, align) = self.size_align_of_unsized(ty, metadata, locals)?;784                    destination.write_from_bytes(self, &align.to_le_bytes())785                }786            }787            "type_name" => {788                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {789                    return Err(MirEvalError::InternalError(790                        "type_name generic arg is not provided".into(),791                    ));792                };793                let ty_name = match ty.display_source_code(794                    self.db,795                    locals.body.owner.module(self.db),796                    true,797                ) {798                    Ok(ty_name) => ty_name,799                    // Fallback to human readable display in case of `Err`. Ideally we want to use `display_source_code` to800                    // render full paths.801                    Err(_) => {802                        let krate = locals.body.owner.krate(self.db);803                        ty.display(self.db, DisplayTarget::from_crate(self.db, krate)).to_string()804                    }805                };806                let len = ty_name.len();807                let addr = self.heap_allocate(len, 1)?;808                self.write_memory(addr, ty_name.as_bytes())?;809                destination.slice(0..self.ptr_size()).write_from_bytes(self, &addr.to_bytes())?;810                destination811                    .slice(self.ptr_size()..2 * self.ptr_size())812                    .write_from_bytes(self, &len.to_le_bytes())813            }814            "needs_drop" => {815                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {816                    return Err(MirEvalError::InternalError(817                        "size_of generic arg is not provided".into(),818                    ));819                };820                let result = match has_drop_glue(&self.infcx, ty, self.param_env.param_env) {821                    DropGlue::HasDropGlue => true,822                    DropGlue::None => false,823                    DropGlue::DependOnParams => {824                        never!("should be fully monomorphized now");825                        true826                    }827                };828                destination.write_from_bytes(self, &[u8::from(result)])829            }830            "ptr_guaranteed_cmp" => {831                // FIXME: this is wrong for const eval, it should return 2 in some832                // cases.833                let [lhs, rhs] = args else {834                    return Err(MirEvalError::InternalError(835                        "ptr_guaranteed_cmp args are not provided".into(),836                    ));837                };838                let ans = lhs.get(self)? == rhs.get(self)?;839                destination.write_from_bytes(self, &[u8::from(ans)])840            }841            "saturating_add" | "saturating_sub" => {842                let [lhs, rhs] = args else {843                    return Err(MirEvalError::InternalError(844                        "saturating_add args are not provided".into(),845                    ));846                };847                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));848                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));849                let ans = match name {850                    "saturating_add" => lhs.saturating_add(rhs),851                    "saturating_sub" => lhs.saturating_sub(rhs),852                    _ => unreachable!(),853                };854                let bits = destination.size * 8;855                // FIXME: signed856                let is_signed = false;857                let mx: u128 = if is_signed { (1 << (bits - 1)) - 1 } else { (1 << bits) - 1 };858                // FIXME: signed859                let mn: u128 = 0;860                let ans = cmp::min(mx, cmp::max(mn, ans));861                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])862            }863            "wrapping_add" | "unchecked_add" => {864                let [lhs, rhs] = args else {865                    return Err(MirEvalError::InternalError(866                        "wrapping_add args are not provided".into(),867                    ));868                };869                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));870                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));871                let ans = lhs.wrapping_add(rhs);872                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])873            }874            "ptr_offset_from_unsigned" | "ptr_offset_from" => {875                let [lhs, rhs] = args else {876                    return Err(MirEvalError::InternalError(877                        "wrapping_sub args are not provided".into(),878                    ));879                };880                let lhs = i128::from_le_bytes(pad16(lhs.get(self)?, false));881                let rhs = i128::from_le_bytes(pad16(rhs.get(self)?, false));882                let ans = lhs.wrapping_sub(rhs);883                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {884                    return Err(MirEvalError::InternalError(885                        "ptr_offset_from generic arg is not provided".into(),886                    ));887                };888                let size = self.size_of_sized(ty, locals, "ptr_offset_from arg")? as i128;889                let ans = ans / size;890                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])891            }892            "wrapping_sub" | "unchecked_sub" => {893                let [lhs, rhs] = args else {894                    return Err(MirEvalError::InternalError(895                        "wrapping_sub args are not provided".into(),896                    ));897                };898                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));899                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));900                let ans = lhs.wrapping_sub(rhs);901                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])902            }903            "wrapping_mul" | "unchecked_mul" => {904                let [lhs, rhs] = args else {905                    return Err(MirEvalError::InternalError(906                        "wrapping_mul args are not provided".into(),907                    ));908                };909                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));910                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));911                let ans = lhs.wrapping_mul(rhs);912                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])913            }914            "wrapping_shl" | "unchecked_shl" => {915                // FIXME: signed916                let [lhs, rhs] = args else {917                    return Err(MirEvalError::InternalError(918                        "unchecked_shl args are not provided".into(),919                    ));920                };921                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));922                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));923                let ans = lhs.wrapping_shl(rhs as u32);924                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])925            }926            "wrapping_shr" | "unchecked_shr" => {927                // FIXME: signed928                let [lhs, rhs] = args else {929                    return Err(MirEvalError::InternalError(930                        "unchecked_shr args are not provided".into(),931                    ));932                };933                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));934                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));935                let ans = lhs.wrapping_shr(rhs as u32);936                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])937            }938            "unchecked_rem" => {939                // FIXME: signed940                let [lhs, rhs] = args else {941                    return Err(MirEvalError::InternalError(942                        "unchecked_rem args are not provided".into(),943                    ));944                };945                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));946                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));947                let ans = lhs.checked_rem(rhs).ok_or_else(|| {948                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())949                })?;950                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])951            }952            "unchecked_div" | "exact_div" => {953                // FIXME: signed954                let [lhs, rhs] = args else {955                    return Err(MirEvalError::InternalError(956                        "unchecked_div args are not provided".into(),957                    ));958                };959                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));960                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));961                let ans = lhs.checked_div(rhs).ok_or_else(|| {962                    MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned())963                })?;964                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])965            }966            "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => {967                let [lhs, rhs] = args else {968                    return Err(MirEvalError::InternalError(969                        "const_eval_select args are not provided".into(),970                    ));971                };972                let result_ty = Ty::new_tup_from_iter(973                    self.interner(),974                    [lhs.ty, Ty::new_bool(self.interner())].into_iter(),975                );976                let op_size = self.size_of_sized(lhs.ty, locals, "operand of add_with_overflow")?;977                let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false));978                let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false));979                let (ans, u128overflow) = match name {980                    "add_with_overflow" => lhs.overflowing_add(rhs),981                    "sub_with_overflow" => lhs.overflowing_sub(rhs),982                    "mul_with_overflow" => lhs.overflowing_mul(rhs),983                    _ => unreachable!(),984                };985                let is_overflow = u128overflow986                    || ans.to_le_bytes()[op_size..].iter().any(|&it| it != 0 && it != 255);987                let is_overflow = vec![u8::from(is_overflow)];988                let layout = self.layout(result_ty)?;989                let result = self.construct_with_layout(990                    layout.size.bytes_usize(),991                    &layout,992                    None,993                    [ans.to_le_bytes()[0..op_size].to_vec(), is_overflow]994                        .into_iter()995                        .map(IntervalOrOwned::Owned),996                )?;997                destination.write_from_bytes(self, &result)998            }999            "copy" | "copy_nonoverlapping" => {1000                let [src, dst, offset] = args else {1001                    return Err(MirEvalError::InternalError(1002                        "copy_nonoverlapping args are not provided".into(),1003                    ));1004                };1005                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1006                    return Err(MirEvalError::InternalError(1007                        "copy_nonoverlapping generic arg is not provided".into(),1008                    ));1009                };1010                let src = Address::from_bytes(src.get(self)?)?;1011                let dst = Address::from_bytes(dst.get(self)?)?;1012                let offset = from_bytes!(usize, offset.get(self)?);1013                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;1014                let size = offset * size;1015                let src = Interval { addr: src, size };1016                let dst = Interval { addr: dst, size };1017                dst.write_from_interval(self, src)1018            }1019            "offset" | "arith_offset" => {1020                let [ptr, offset] = args else {1021                    return Err(MirEvalError::InternalError("offset args are not provided".into()));1022                };1023                let ty = if name == "offset" {1024                    let Some(ty0) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1025                        return Err(MirEvalError::InternalError(1026                            "offset generic arg is not provided".into(),1027                        ));1028                    };1029                    let Some(ty1) = generic_args.as_slice().get(1).and_then(|it| it.ty()) else {1030                        return Err(MirEvalError::InternalError(1031                            "offset generic arg is not provided".into(),1032                        ));1033                    };1034                    if !matches!(1035                        ty1.kind(),1036                        TyKind::Int(rustc_type_ir::IntTy::Isize)1037                            | TyKind::Uint(rustc_type_ir::UintTy::Usize)1038                    ) {1039                        return Err(MirEvalError::InternalError(1040                            "offset generic arg is not usize or isize".into(),1041                        ));1042                    }1043                    match ty0.kind() {1044                        TyKind::RawPtr(ty, _) => ty,1045                        _ => {1046                            return Err(MirEvalError::InternalError(1047                                "offset generic arg is not a raw pointer".into(),1048                            ));1049                        }1050                    }1051                } else {1052                    let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1053                        return Err(MirEvalError::InternalError(1054                            "arith_offset generic arg is not provided".into(),1055                        ));1056                    };1057                    ty1058                };1059                let ptr = u128::from_le_bytes(pad16(ptr.get(self)?, false));1060                let offset = u128::from_le_bytes(pad16(offset.get(self)?, false));1061                let size = self.size_of_sized(ty, locals, "offset ptr type")? as u128;1062                let ans = ptr + offset * size;1063                destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size])1064            }1065            "assert_inhabited" | "assert_zero_valid" | "assert_uninit_valid" | "assume" => {1066                // FIXME: We should actually implement these checks1067                Ok(())1068            }1069            "forget" => {1070                // We don't call any drop glue yet, so there is nothing here1071                Ok(())1072            }1073            "transmute" | "transmute_unchecked" => {1074                let [arg] = args else {1075                    return Err(MirEvalError::InternalError(1076                        "transmute arg is not provided".into(),1077                    ));1078                };1079                destination.write_from_interval(self, arg.interval)1080            }1081            "ctpop" => {1082                let [arg] = args else {1083                    return Err(MirEvalError::InternalError("ctpop arg is not provided".into()));1084                };1085                let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).count_ones();1086                destination1087                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])1088            }1089            "ctlz" | "ctlz_nonzero" => {1090                let [arg] = args else {1091                    return Err(MirEvalError::InternalError("ctlz arg is not provided".into()));1092                };1093                let result =1094                    u128::from_le_bytes(pad16(arg.get(self)?, false)).leading_zeros() as usize;1095                let result = result - (128 - arg.interval.size * 8);1096                destination1097                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])1098            }1099            "cttz" | "cttz_nonzero" => {1100                let [arg] = args else {1101                    return Err(MirEvalError::InternalError("cttz arg is not provided".into()));1102                };1103                let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).trailing_zeros();1104                destination1105                    .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size])1106            }1107            "rotate_left" => {1108                let [lhs, rhs] = args else {1109                    return Err(MirEvalError::InternalError(1110                        "rotate_left args are not provided".into(),1111                    ));1112                };1113                let lhs = &lhs.get(self)?[0..destination.size];1114                let rhs = rhs.get(self)?[0] as u32;1115                match destination.size {1116                    1 => {1117                        let r = from_bytes!(u8, lhs).rotate_left(rhs);1118                        destination.write_from_bytes(self, &r.to_le_bytes())1119                    }1120                    2 => {1121                        let r = from_bytes!(u16, lhs).rotate_left(rhs);1122                        destination.write_from_bytes(self, &r.to_le_bytes())1123                    }1124                    4 => {1125                        let r = from_bytes!(u32, lhs).rotate_left(rhs);1126                        destination.write_from_bytes(self, &r.to_le_bytes())1127                    }1128                    8 => {1129                        let r = from_bytes!(u64, lhs).rotate_left(rhs);1130                        destination.write_from_bytes(self, &r.to_le_bytes())1131                    }1132                    16 => {1133                        let r = from_bytes!(u128, lhs).rotate_left(rhs);1134                        destination.write_from_bytes(self, &r.to_le_bytes())1135                    }1136                    s => not_supported!("destination with size {s} for rotate_left"),1137                }1138            }1139            "rotate_right" => {1140                let [lhs, rhs] = args else {1141                    return Err(MirEvalError::InternalError(1142                        "rotate_right args are not provided".into(),1143                    ));1144                };1145                let lhs = &lhs.get(self)?[0..destination.size];1146                let rhs = rhs.get(self)?[0] as u32;1147                match destination.size {1148                    1 => {1149                        let r = from_bytes!(u8, lhs).rotate_right(rhs);1150                        destination.write_from_bytes(self, &r.to_le_bytes())1151                    }1152                    2 => {1153                        let r = from_bytes!(u16, lhs).rotate_right(rhs);1154                        destination.write_from_bytes(self, &r.to_le_bytes())1155                    }1156                    4 => {1157                        let r = from_bytes!(u32, lhs).rotate_right(rhs);1158                        destination.write_from_bytes(self, &r.to_le_bytes())1159                    }1160                    8 => {1161                        let r = from_bytes!(u64, lhs).rotate_right(rhs);1162                        destination.write_from_bytes(self, &r.to_le_bytes())1163                    }1164                    16 => {1165                        let r = from_bytes!(u128, lhs).rotate_right(rhs);1166                        destination.write_from_bytes(self, &r.to_le_bytes())1167                    }1168                    s => not_supported!("destination with size {s} for rotate_right"),1169                }1170            }1171            "discriminant_value" => {1172                let [arg] = args else {1173                    return Err(MirEvalError::InternalError(1174                        "discriminant_value arg is not provided".into(),1175                    ));1176                };1177                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1178                    return Err(MirEvalError::InternalError(1179                        "discriminant_value generic arg is not provided".into(),1180                    ));1181                };1182                let addr = Address::from_bytes(arg.get(self)?)?;1183                let size = self.size_of_sized(ty, locals, "discriminant_value ptr type")?;1184                let interval = Interval { addr, size };1185                let r = self.compute_discriminant(ty, interval.get(self)?)?;1186                destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])1187            }1188            "const_eval_select" => {1189                let [tuple, const_fn, _] = args else {1190                    return Err(MirEvalError::InternalError(1191                        "const_eval_select args are not provided".into(),1192                    ));1193                };1194                let mut args = vec![const_fn.clone()];1195                let TyKind::Tuple(fields) = tuple.ty.kind() else {1196                    return Err(MirEvalError::InternalError(1197                        "const_eval_select arg[0] is not a tuple".into(),1198                    ));1199                };1200                let layout = self.layout(tuple.ty)?;1201                for (i, field) in fields.iter().enumerate() {1202                    let offset = layout.fields.offset(i).bytes_usize();1203                    let addr = tuple.interval.addr.offset(offset);1204                    args.push(IntervalAndTy::new(addr, field, self, locals)?);1205                }1206                if let Some(def) = self.lang_items().FnOnce_call_once {1207                    self.exec_fn_trait(1208                        def,1209                        &args,1210                        // FIXME: wrong for manual impls of `FnOnce`1211                        GenericArgs::empty(self.interner()),1212                        locals,1213                        destination,1214                        None,1215                        span,1216                    )?;1217                    return Ok(true);1218                }1219                not_supported!("FnOnce was not available for executing const_eval_select");1220            }1221            "read_via_copy" | "volatile_load" => {1222                let [arg] = args else {1223                    return Err(MirEvalError::InternalError(1224                        "read_via_copy args are not provided".into(),1225                    ));1226                };1227                let addr = Address::from_bytes(arg.interval.get(self)?)?;1228                destination.write_from_interval(self, Interval { addr, size: destination.size })1229            }1230            "write_via_move" => {1231                let [ptr, val] = args else {1232                    return Err(MirEvalError::InternalError(1233                        "write_via_move args are not provided".into(),1234                    ));1235                };1236                let dst = Address::from_bytes(ptr.get(self)?)?;1237                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1238                    return Err(MirEvalError::InternalError(1239                        "write_via_copy generic arg is not provided".into(),1240                    ));1241                };1242                let size = self.size_of_sized(ty, locals, "write_via_move ptr type")?;1243                Interval { addr: dst, size }.write_from_interval(self, val.interval)?;1244                Ok(())1245            }1246            "write_bytes" => {1247                let [dst, val, count] = args else {1248                    return Err(MirEvalError::InternalError(1249                        "write_bytes args are not provided".into(),1250                    ));1251                };1252                let count = from_bytes!(usize, count.get(self)?);1253                let val = from_bytes!(u8, val.get(self)?);1254                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1255                    return Err(MirEvalError::InternalError(1256                        "write_bytes generic arg is not provided".into(),1257                    ));1258                };1259                let dst = Address::from_bytes(dst.get(self)?)?;1260                let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?;1261                let size = count * size;1262                self.write_memory_using_ref(dst, size)?.fill(val);1263                Ok(())1264            }1265            "ptr_metadata" => {1266                let [ptr] = args else {1267                    return Err(MirEvalError::InternalError(1268                        "ptr_metadata args are not provided".into(),1269                    ));1270                };1271                let arg = ptr.interval.get(self)?.to_owned();1272                let metadata = &arg[self.ptr_size()..];1273                destination.write_from_bytes(self, metadata)?;1274                Ok(())1275            }1276            "three_way_compare" => {1277                let [lhs, rhs] = args else {1278                    return Err(MirEvalError::InternalError(1279                        "three_way_compare args are not provided".into(),1280                    ));1281                };1282                let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1283                    return Err(MirEvalError::InternalError(1284                        "three_way_compare generic arg is not provided".into(),1285                    ));1286                };1287                let signed = match ty.kind() {1288                    TyKind::Int(_) => true,1289                    TyKind::Uint(_) => false,1290                    _ => {1291                        return Err(MirEvalError::InternalError(1292                            "three_way_compare expects an integral type".into(),1293                        ));1294                    }1295                };1296                let rhs = rhs.get(self)?;1297                let lhs = lhs.get(self)?;1298                let mut result = Ordering::Equal;1299                for (l, r) in lhs.iter().zip(rhs).rev() {1300                    let it = l.cmp(r);1301                    if it != Ordering::Equal {1302                        result = it;1303                        break;1304                    }1305                }1306                if signed1307                    && let Some((&l, &r)) = lhs.iter().zip(rhs).next_back()1308                    && l != r1309                {1310                    result = (l as i8).cmp(&(r as i8));1311                }1312                if let Some(e) = self.lang_items().Ordering {1313                    let ty = self.db.ty(e.into()).skip_binder();1314                    let r = self.compute_discriminant(ty, &[result as i8 as u8])?;1315                    destination.write_from_bytes(self, &r.to_le_bytes()[0..destination.size])?;1316                    Ok(())1317                } else {1318                    Err(MirEvalError::InternalError("Ordering enum not found".into()))1319                }1320            }1321            "aggregate_raw_ptr" => {1322                let [data, meta] = args else {1323                    return Err(MirEvalError::InternalError(1324                        "aggregate_raw_ptr args are not provided".into(),1325                    ));1326                };1327                destination.write_from_interval(self, data.interval)?;1328                Interval {1329                    addr: destination.addr.offset(data.interval.size),1330                    size: destination.size - data.interval.size,1331                }1332                .write_from_interval(self, meta.interval)?;1333                Ok(())1334            }1335            _ if needs_override => not_supported!("intrinsic {name} is not implemented"),1336            _ => return Ok(false),1337        }1338        .map(|()| true)1339    }13401341    fn size_align_of_unsized(1342        &mut self,1343        ty: Ty<'db>,1344        metadata: Interval,1345        locals: &Locals<'a>,1346    ) -> Result<'db, (usize, usize)> {1347        Ok(match ty.kind() {1348            TyKind::Str => (from_bytes!(usize, metadata.get(self)?), 1),1349            TyKind::Slice(inner) => {1350                let len = from_bytes!(usize, metadata.get(self)?);1351                let (size, align) = self.size_align_of_sized(inner, locals, "slice inner type")?;1352                (size * len, align)1353            }1354            TyKind::Dynamic(..) => self.size_align_of_sized(1355                self.vtable_map.ty_of_bytes(metadata.get(self)?)?,1356                locals,1357                "dyn concrete type",1358            )?,1359            TyKind::Adt(adt_def, subst) => {1360                let id = adt_def.def_id();1361                let layout = self.layout_adt(id, subst)?;1362                let id = match id {1363                    AdtId::StructId(s) => s,1364                    _ => not_supported!("unsized enum or union"),1365                };1366                let field_types = self.db.field_types(id.into());1367                let last_field_ty = field_types1368                    .iter()1369                    .next_back()1370                    .unwrap()1371                    .11372                    .ty()1373                    .instantiate(self.interner(), subst)1374                    .skip_norm_wip();1375                let sized_part_size =1376                    layout.fields.offset(field_types.iter().count() - 1).bytes_usize();1377                let sized_part_align = layout.align.bytes() as usize;1378                let (unsized_part_size, unsized_part_align) =1379                    self.size_align_of_unsized(last_field_ty, metadata, locals)?;1380                let align = sized_part_align.max(unsized_part_align) as isize;1381                let size = (sized_part_size + unsized_part_size) as isize;1382                // Must add any necessary padding to `size`1383                // (to make it a multiple of `align`) before returning it.1384                //1385                // Namely, the returned size should be, in C notation:1386                //1387                //   `size + ((size & (align-1)) ? align : 0)`1388                //1389                // emulated via the semi-standard fast bit trick:1390                //1391                //   `(size + (align-1)) & -align`1392                let size = (size + (align - 1)) & (-align);1393                (size as usize, align as usize)1394            }1395            _ => not_supported!("unsized type other than str, slice, struct and dyn"),1396        })1397    }13981399    fn exec_atomic_intrinsic(1400        &mut self,1401        name: &str,1402        args: &[IntervalAndTy<'db>],1403        generic_args: GenericArgs<'db>,1404        destination: Interval,1405        locals: &Locals<'a>,1406        _span: MirSpan,1407    ) -> Result<'db, ()> {1408        // We are a single threaded runtime with no UB checking and no optimization, so1409        // we can implement atomic intrinsics as normal functions.14101411        if name.starts_with("singlethreadfence_") || name.starts_with("fence_") {1412            return Ok(());1413        }14141415        // The rest of atomic intrinsics have exactly one generic arg14161417        let Some(ty) = generic_args.as_slice().first().and_then(|it| it.ty()) else {1418            return Err(MirEvalError::InternalError(1419                "atomic intrinsic generic arg is not provided".into(),1420            ));1421        };1422        let Some(arg0) = args.first() else {1423            return Err(MirEvalError::InternalError(1424                "atomic intrinsic arg0 is not provided".into(),1425            ));1426        };1427        let arg0_addr = Address::from_bytes(arg0.get(self)?)?;1428        let arg0_interval =1429            Interval::new(arg0_addr, self.size_of_sized(ty, locals, "atomic intrinsic type arg")?);1430        if name.starts_with("load_") {1431            return destination.write_from_interval(self, arg0_interval);1432        }1433        let Some(arg1) = args.get(1) else {1434            return Err(MirEvalError::InternalError(1435                "atomic intrinsic arg1 is not provided".into(),1436            ));1437        };1438        if name.starts_with("store_") {1439            return arg0_interval.write_from_interval(self, arg1.interval);1440        }1441        if name.starts_with("xchg_") {1442            destination.write_from_interval(self, arg0_interval)?;1443            return arg0_interval.write_from_interval(self, arg1.interval);1444        }1445        if name.starts_with("xadd_") {1446            destination.write_from_interval(self, arg0_interval)?;1447            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1448            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1449            let ans = lhs.wrapping_add(rhs);1450            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1451        }1452        if name.starts_with("xsub_") {1453            destination.write_from_interval(self, arg0_interval)?;1454            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1455            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1456            let ans = lhs.wrapping_sub(rhs);1457            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1458        }1459        if name.starts_with("and_") {1460            destination.write_from_interval(self, arg0_interval)?;1461            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1462            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1463            let ans = lhs & rhs;1464            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1465        }1466        if name.starts_with("or_") {1467            destination.write_from_interval(self, arg0_interval)?;1468            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1469            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1470            let ans = lhs | rhs;1471            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1472        }1473        if name.starts_with("xor_") {1474            destination.write_from_interval(self, arg0_interval)?;1475            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1476            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1477            let ans = lhs ^ rhs;1478            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1479        }1480        if name.starts_with("nand_") {1481            destination.write_from_interval(self, arg0_interval)?;1482            let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false));1483            let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false));1484            let ans = !(lhs & rhs);1485            return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]);1486        }1487        let Some(arg2) = args.get(2) else {1488            return Err(MirEvalError::InternalError(1489                "atomic intrinsic arg2 is not provided".into(),1490            ));1491        };1492        if name.starts_with("cxchg_") || name.starts_with("cxchgweak_") {1493            let dest = if arg1.get(self)? == arg0_interval.get(self)? {1494                arg0_interval.write_from_interval(self, arg2.interval)?;1495                (arg1.interval, true)1496            } else {1497                (arg0_interval, false)1498            };1499            let result_ty = Ty::new_tup_from_iter(1500                self.interner(),1501                [ty, Ty::new_bool(self.interner())].into_iter(),1502            );1503            let layout = self.layout(result_ty)?;1504            let result = self.construct_with_layout(1505                layout.size.bytes_usize(),1506                &layout,1507                None,1508                [IntervalOrOwned::Borrowed(dest.0), IntervalOrOwned::Owned(vec![u8::from(dest.1)])]1509                    .into_iter(),1510            )?;1511            return destination.write_from_bytes(self, &result);1512        }1513        not_supported!("unknown atomic intrinsic {name}");1514    }1515}