Skip to main content

rustc_codegen_llvm/
asm.rs

1use std::assert_matches;
2
3use rustc_abi::{BackendRepr, Float, Integer, Primitive, Scalar};
4use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
5use rustc_codegen_ssa::mir::operand::OperandValue;
6use rustc_codegen_ssa::traits::*;
7use rustc_data_structures::fx::FxHashMap;
8use rustc_middle::ty::Instance;
9use rustc_middle::ty::layout::TyAndLayout;
10use rustc_middle::{bug, span_bug};
11use rustc_span::{Pos, Span, Symbol, sym};
12use rustc_target::asm::*;
13use smallvec::SmallVec;
14use tracing::debug;
15
16use crate::attributes;
17use crate::builder::Builder;
18use crate::common::Funclet;
19use crate::context::CodegenCx;
20use crate::llvm::{self, ToLlvmBool, Type, Value};
21use crate::type_of::LayoutLlvmExt;
22
23impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
24    fn codegen_inline_asm(
25        &mut self,
26        template: &[InlineAsmTemplatePiece],
27        operands: &[InlineAsmOperandRef<'tcx, Self>],
28        options: InlineAsmOptions,
29        line_spans: &[Span],
30        instance: Instance<'_>,
31        dest: Option<Self::BasicBlock>,
32        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
33    ) {
34        let asm_arch = self.tcx.sess.asm_arch.unwrap();
35
36        // Collect the types of output operands
37        let mut constraints = ::alloc::vec::Vec::new()vec![];
38        let mut clobbers = ::alloc::vec::Vec::new()vec![];
39        let mut output_types = ::alloc::vec::Vec::new()vec![];
40        let mut op_idx = FxHashMap::default();
41        let mut clobbered_x87 = false;
42        for (idx, op) in operands.iter().enumerate() {
43            match *op {
44                InlineAsmOperandRef::Out { reg, late, place } => {
45                    let is_target_supported = |reg_class: InlineAsmRegClass| {
46                        for &(_, feature) in reg_class.supported_types(asm_arch, true).as_ref() {
47                            if let Some(feature) = feature {
48                                if self
49                                    .tcx
50                                    .asm_target_features(instance.def_id())
51                                    .contains(&feature)
52                                {
53                                    return true;
54                                }
55                            } else {
56                                // Register class is unconditionally supported
57                                return true;
58                            }
59                        }
60                        false
61                    };
62
63                    let mut layout = None;
64                    let ty = if let Some(ref place) = place {
65                        layout = Some(&place.layout);
66                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
67                    } else if #[allow(non_exhaustive_omitted_patterns)] match reg.reg_class() {
    InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg |
        X86InlineAsmRegClass::x87_reg) => true,
    _ => false,
}matches!(
68                        reg.reg_class(),
69                        InlineAsmRegClass::X86(
70                            X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
71                        )
72                    ) {
73                        // Special handling for x87/mmx registers: we always
74                        // clobber the whole set if one register is marked as
75                        // clobbered. This is due to the way LLVM handles the
76                        // FP stack in inline assembly.
77                        if !clobbered_x87 {
78                            clobbered_x87 = true;
79                            clobbers.push("~{st}".to_string());
80                            for i in 1..=7 {
81                                clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{{st({0})}}", i))
    })format!("~{{st({})}}", i));
82                            }
83                        }
84                        continue;
85                    } else if !is_target_supported(reg.reg_class())
86                        || reg.reg_class().is_clobber_only(asm_arch, true)
87                    {
88                        // We turn discarded outputs into clobber constraints
89                        // if the target feature needed by the register class is
90                        // disabled. This is necessary otherwise LLVM will try
91                        // to actually allocate a register for the dummy output.
92                        {
    match reg {
        InlineAsmRegOrRegClass::Reg(_) => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "InlineAsmRegOrRegClass::Reg(_)",
                ::core::option::Option::None);
        }
    }
};assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_));
93                        clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{0}", reg_to_llvm(reg, None)))
    })format!("~{}", reg_to_llvm(reg, None)));
94                        continue;
95                    } else {
96                        // If the output is discarded, we don't really care what
97                        // type is used. We're just using this to tell LLVM to
98                        // reserve the register.
99                        dummy_output_type(self.cx, reg.reg_class())
100                    };
101                    output_types.push(ty);
102                    op_idx.insert(idx, constraints.len());
103                    let prefix = if late { "=" } else { "=&" };
104                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, layout)))
    })format!("{}{}", prefix, reg_to_llvm(reg, layout)));
105                }
106                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
107                    let layout = if let Some(ref out_place) = out_place {
108                        &out_place.layout
109                    } else {
110                        // LLVM required tied operands to have the same type,
111                        // so we just use the type of the input.
112                        &in_value.layout
113                    };
114                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
115                    output_types.push(ty);
116                    op_idx.insert(idx, constraints.len());
117                    let prefix = if late { "=" } else { "=&" };
118                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, Some(layout))))
    })format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
119                }
120                _ => {}
121            }
122        }
123
124        // Collect input operands
125        let mut inputs = ::alloc::vec::Vec::new()vec![];
126        for (idx, op) in operands.iter().enumerate() {
127            match *op {
128                InlineAsmOperandRef::In { reg, value } => {
129                    let llval = llvm_fixup_input(
130                        self,
131                        value.immediate(),
132                        reg.reg_class(),
133                        &value.layout,
134                        instance,
135                    );
136                    inputs.push(llval);
137                    op_idx.insert(idx, constraints.len());
138                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
139                }
140                InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
141                    let value = llvm_fixup_input(
142                        self,
143                        in_value.immediate(),
144                        reg.reg_class(),
145                        &in_value.layout,
146                        instance,
147                    );
148                    inputs.push(value);
149
150                    // In the case of fixed registers, we have the choice of
151                    // either using a tied operand or duplicating the constraint.
152                    // We prefer the latter because it matches the behavior of
153                    // Clang.
154                    if late && #[allow(non_exhaustive_omitted_patterns)] match reg {
    InlineAsmRegOrRegClass::Reg(_) => true,
    _ => false,
}matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
155                        constraints.push(reg_to_llvm(reg, Some(&in_value.layout)));
156                    } else {
157                        constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}", op_idx[&idx]))
    })format!("{}", op_idx[&idx]));
158                    }
159                }
160                InlineAsmOperandRef::SymFn { instance } => {
161                    inputs.push(self.cx.get_fn(instance));
162                    op_idx.insert(idx, constraints.len());
163                    constraints.push("s".to_string());
164                }
165                InlineAsmOperandRef::SymStatic { def_id } => {
166                    inputs.push(self.cx.get_static(def_id));
167                    op_idx.insert(idx, constraints.len());
168                    constraints.push("s".to_string());
169                }
170                _ => {}
171            }
172        }
173
174        // Build the template string
175        let mut labels = ::alloc::vec::Vec::new()vec![];
176        let mut template_str = String::new();
177        for piece in template {
178            match *piece {
179                InlineAsmTemplatePiece::String(ref s) => {
180                    if s.contains('$') {
181                        for c in s.chars() {
182                            if c == '$' {
183                                template_str.push_str("$$");
184                            } else {
185                                template_str.push(c);
186                            }
187                        }
188                    } else {
189                        template_str.push_str(s)
190                    }
191                }
192                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
193                    match operands[operand_idx] {
194                        InlineAsmOperandRef::In { reg, .. }
195                        | InlineAsmOperandRef::Out { reg, .. }
196                        | InlineAsmOperandRef::InOut { reg, .. } => {
197                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
198                            if let Some(modifier) = modifier {
199                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:{1}}}",
                op_idx[&operand_idx], modifier))
    })format!(
200                                    "${{{}:{}}}",
201                                    op_idx[&operand_idx], modifier
202                                ));
203                            } else {
204                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}}}", op_idx[&operand_idx]))
    })format!("${{{}}}", op_idx[&operand_idx]));
205                            }
206                        }
207                        InlineAsmOperandRef::Const { ref string } => {
208                            // Const operands get injected directly into the template
209                            template_str.push_str(string);
210                        }
211                        InlineAsmOperandRef::SymFn { .. }
212                        | InlineAsmOperandRef::SymStatic { .. } => {
213                            // Only emit the raw symbol name
214                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:c}}", op_idx[&operand_idx]))
    })format!("${{{}:c}}", op_idx[&operand_idx]));
215                        }
216                        InlineAsmOperandRef::Label { label } => {
217                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:l}}", constraints.len()))
    })format!("${{{}:l}}", constraints.len()));
218                            constraints.push("!i".to_owned());
219                            labels.push(label);
220                        }
221                    }
222                }
223            }
224        }
225
226        constraints.append(&mut clobbers);
227        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
228            match asm_arch {
229                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
230                    constraints.push("~{cc}".to_string());
231                }
232                InlineAsmArch::Amdgpu => {}
233                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
234                    constraints.extend_from_slice(&[
235                        "~{dirflag}".to_string(),
236                        "~{fpsr}".to_string(),
237                        "~{flags}".to_string(),
238                    ]);
239                }
240                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
241                    constraints.extend_from_slice(&[
242                        "~{fflags}".to_string(),
243                        "~{vtype}".to_string(),
244                        "~{vl}".to_string(),
245                        "~{vxsat}".to_string(),
246                        "~{vxrm}".to_string(),
247                    ]);
248                }
249                InlineAsmArch::Avr => {
250                    constraints.push("~{sreg}".to_string());
251                }
252                InlineAsmArch::Nvptx64 => {}
253                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
254                InlineAsmArch::Hexagon => {}
255                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
256                    constraints.extend_from_slice(&[
257                        "~{$fcc0}".to_string(),
258                        "~{$fcc1}".to_string(),
259                        "~{$fcc2}".to_string(),
260                        "~{$fcc3}".to_string(),
261                        "~{$fcc4}".to_string(),
262                        "~{$fcc5}".to_string(),
263                        "~{$fcc6}".to_string(),
264                        "~{$fcc7}".to_string(),
265                    ]);
266                }
267                InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
268                InlineAsmArch::S390x => {
269                    constraints.push("~{cc}".to_string());
270                }
271                InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => {
272                    // In LLVM, ~{icc} represents icc and xcc in 64-bit code.
273                    // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64
274                    constraints.push("~{icc}".to_string());
275                    constraints.push("~{fcc0}".to_string());
276                    constraints.push("~{fcc1}".to_string());
277                    constraints.push("~{fcc2}".to_string());
278                    constraints.push("~{fcc3}".to_string());
279                }
280                InlineAsmArch::SpirV => {}
281                InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
282                InlineAsmArch::Xtensa => {}
283                InlineAsmArch::Bpf => {}
284                InlineAsmArch::Msp430 => {
285                    constraints.push("~{sr}".to_string());
286                }
287                InlineAsmArch::M68k => {
288                    constraints.push("~{ccr}".to_string());
289                }
290                InlineAsmArch::CSKY => {
291                    constraints.push("~{psr}".to_string());
292                }
293            }
294        }
295        if !options.contains(InlineAsmOptions::NOMEM) {
296            // This is actually ignored by LLVM, but it's probably best to keep
297            // it just in case. LLVM instead uses the ReadOnly/ReadNone
298            // attributes on the call instruction to optimize.
299            constraints.push("~{memory}".to_string());
300        }
301        let volatile = !options.contains(InlineAsmOptions::PURE);
302        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
303        let output_type = match &output_types[..] {
304            [] => self.type_void(),
305            [ty] => ty,
306            tys => self.type_struct(tys, false),
307        };
308        let dialect = match asm_arch {
309            InlineAsmArch::X86 | InlineAsmArch::X86_64
310                if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
311            {
312                llvm::AsmDialect::Intel
313            }
314            _ => llvm::AsmDialect::Att,
315        };
316        let result = inline_asm_call(
317            self,
318            &template_str,
319            &constraints.join(","),
320            &inputs,
321            output_type,
322            &labels,
323            volatile,
324            alignstack,
325            dialect,
326            line_spans,
327            options.contains(InlineAsmOptions::MAY_UNWIND),
328            dest,
329            catch_funclet,
330        )
331        .unwrap_or_else(|| ::rustc_middle::util::bug::span_bug_fmt(line_spans[0],
    format_args!("LLVM asm constraint validation failed"))span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
332
333        let mut attrs = SmallVec::<[_; 2]>::new();
334        if options.contains(InlineAsmOptions::PURE) {
335            if options.contains(InlineAsmOptions::NOMEM) {
336                attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
337            } else if options.contains(InlineAsmOptions::READONLY) {
338                attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
339            }
340            attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
341        } else if options.contains(InlineAsmOptions::NOMEM) {
342            attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
343        } else if options.contains(InlineAsmOptions::READONLY) {
344            attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx));
345        }
346        attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
347
348        // Write results to outputs. We need to do this for all possible control flow.
349        //
350        // Note that `dest` maybe populated with unreachable_block when asm goto with outputs
351        // is used (because we need to codegen callbr which always needs a destination), so
352        // here we use the NORETURN option to determine if `dest` should be used.
353        for block in (if options.contains(InlineAsmOptions::NORETURN) { None } else { Some(dest) })
354            .into_iter()
355            .chain(labels.iter().copied().map(Some))
356        {
357            if let Some(block) = block {
358                self.switch_to_block(block);
359            }
360
361            for (idx, op) in operands.iter().enumerate() {
362                if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
363                | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
364                {
365                    let value = if output_types.len() == 1 {
366                        result
367                    } else {
368                        self.extract_value(result, op_idx[&idx] as u64)
369                    };
370                    let value =
371                        llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
372                    OperandValue::Immediate(value).store(self, place);
373                }
374            }
375        }
376    }
377}
378
379impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
380    fn codegen_global_asm(
381        &mut self,
382        template: &[InlineAsmTemplatePiece],
383        operands: &[GlobalAsmOperandRef<'tcx>],
384        options: InlineAsmOptions,
385        _line_spans: &[Span],
386    ) {
387        let asm_arch = self.tcx.sess.asm_arch.unwrap();
388
389        // Build the template string
390        let mut template_str = String::new();
391
392        // On X86 platforms there are two assembly syntaxes. Rust uses intel by default,
393        // but AT&T can be specified explicitly.
394        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) {
395            if options.contains(InlineAsmOptions::ATT_SYNTAX) {
396                template_str.push_str(".att_syntax\n")
397            } else {
398                template_str.push_str(".intel_syntax\n")
399            }
400        }
401
402        for piece in template {
403            match *piece {
404                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
405                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span } => {
406                    use rustc_codegen_ssa::back::symbol_export::escape_symbol_name;
407                    match operands[operand_idx] {
408                        GlobalAsmOperandRef::Const { ref string } => {
409                            // Const operands get injected directly into the
410                            // template. Note that we don't need to escape $
411                            // here unlike normal inline assembly.
412                            template_str.push_str(string);
413                        }
414                        GlobalAsmOperandRef::SymFn { instance } => {
415                            let llval = self.get_fn(instance);
416                            self.add_compiler_used_global(llval);
417                            let symbol = llvm::build_string(|s| unsafe {
418                                llvm::LLVMRustGetMangledName(llval, s);
419                            })
420                            .expect("symbol is not valid UTF-8");
421                            template_str.push_str(&escape_symbol_name(self.tcx, &symbol, span));
422                        }
423                        GlobalAsmOperandRef::SymStatic { def_id } => {
424                            let llval = self
425                                .renamed_statics
426                                .borrow()
427                                .get(&def_id)
428                                .copied()
429                                .unwrap_or_else(|| self.get_static(def_id));
430                            self.add_compiler_used_global(llval);
431                            let symbol = llvm::build_string(|s| unsafe {
432                                llvm::LLVMRustGetMangledName(llval, s);
433                            })
434                            .expect("symbol is not valid UTF-8");
435                            template_str.push_str(&escape_symbol_name(self.tcx, &symbol, span));
436                        }
437                    }
438                }
439            }
440        }
441
442        // Just to play it safe, if intel was used, reset the assembly syntax to att.
443        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
444            && !options.contains(InlineAsmOptions::ATT_SYNTAX)
445        {
446            template_str.push_str("\n.att_syntax\n");
447        }
448
449        llvm::append_module_inline_asm(self.llmod, template_str.as_bytes());
450    }
451
452    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
453        let llval = self.get_fn(instance);
454        llvm::build_string(|s| unsafe {
455            llvm::LLVMRustGetMangledName(llval, s);
456        })
457        .expect("symbol is not valid UTF-8")
458    }
459}
460
461pub(crate) fn inline_asm_call<'ll>(
462    bx: &mut Builder<'_, 'll, '_>,
463    asm: &str,
464    cons: &str,
465    inputs: &[&'ll Value],
466    output: &'ll llvm::Type,
467    labels: &[&'ll llvm::BasicBlock],
468    volatile: bool,
469    alignstack: bool,
470    dia: llvm::AsmDialect,
471    line_spans: &[Span],
472    unwind: bool,
473    dest: Option<&'ll llvm::BasicBlock>,
474    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
475) -> Option<&'ll Value> {
476    let argtys = inputs
477        .iter()
478        .map(|v| {
479            {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:479",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(479u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                let mut iter = __CALLSITE.metadata().fields().iter();
                __CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
                                    ::tracing::__macro_support::Option::Some(&format_args!("Asm Input Type: {0:?}",
                                                    *v) as &dyn Value))])
            });
    } else { ; }
};debug!("Asm Input Type: {:?}", *v);
480            bx.cx.val_ty(*v)
481        })
482        .collect::<Vec<_>>();
483
484    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:484",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(484u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                let mut iter = __CALLSITE.metadata().fields().iter();
                __CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
                                    ::tracing::__macro_support::Option::Some(&format_args!("Asm Output Type: {0:?}",
                                                    output) as &dyn Value))])
            });
    } else { ; }
};debug!("Asm Output Type: {:?}", output);
485    let fty = bx.cx.type_func(&argtys, output);
486
487    // Ask LLVM to verify that the constraints are well-formed.
488    let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) };
489    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:489",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(489u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                let mut iter = __CALLSITE.metadata().fields().iter();
                __CALLSITE.metadata().fields().value_set(&[(&::tracing::__macro_support::Iterator::next(&mut iter).expect("FieldSet corrupted (this is a bug)"),
                                    ::tracing::__macro_support::Option::Some(&format_args!("constraint verification result: {0:?}",
                                                    constraints_ok) as &dyn Value))])
            });
    } else { ; }
};debug!("constraint verification result: {:?}", constraints_ok);
490    if !constraints_ok {
491        // LLVM has detected an issue with our constraints, so bail out.
492        return None;
493    }
494
495    let v = unsafe {
496        llvm::LLVMGetInlineAsm(
497            fty,
498            asm.as_ptr(),
499            asm.len(),
500            cons.as_ptr(),
501            cons.len(),
502            volatile.to_llvm_bool(),
503            alignstack.to_llvm_bool(),
504            dia,
505            unwind.to_llvm_bool(),
506        )
507    };
508
509    let call = if !labels.is_empty() {
510        if !catch_funclet.is_none() {
    ::core::panicking::panic("assertion failed: catch_funclet.is_none()")
};assert!(catch_funclet.is_none());
511        bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None)
512    } else if let Some((catch, funclet)) = catch_funclet {
513        bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None)
514    } else {
515        bx.call(fty, None, None, v, inputs, None, None)
516    };
517
518    // Store mark in a metadata node so we can map LLVM errors
519    // back to source locations. See #17552.
520    let key = "srcloc";
521    let kind = bx.get_md_kind_id(key);
522
523    // `srcloc` contains one 64-bit integer for each line of assembly code,
524    // where the lower 32 bits hold the lo byte position and the upper 32 bits
525    // hold the hi byte position.
526    let mut srcloc = ::alloc::vec::Vec::new()vec![];
527    if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
528        // LLVM inserts an extra line to add the ".intel_syntax", so add
529        // a dummy srcloc entry for it.
530        //
531        // Don't do this if we only have 1 line span since that may be
532        // due to the asm template string coming from a macro. LLVM will
533        // default to the first srcloc for lines that don't have an
534        // associated srcloc.
535        srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0)));
536    }
537    srcloc.extend(line_spans.iter().map(|span| {
538        llvm::LLVMValueAsMetadata(
539            bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
540        )
541    }));
542    bx.cx.set_metadata_node(call, kind, &srcloc);
543
544    Some(call)
545}
546
547/// If the register is an xmm/ymm/zmm register then return its index.
548fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
549    use X86InlineAsmReg::*;
550    match reg {
551        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
552            Some(reg as u32 - xmm0 as u32)
553        }
554        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
555            Some(reg as u32 - ymm0 as u32)
556        }
557        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
558            Some(reg as u32 - zmm0 as u32)
559        }
560        _ => None,
561    }
562}
563
564/// If the register is an AArch64 integer register then return its index.
565fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
566    match reg {
567        InlineAsmReg::AArch64(r) => r.reg_index(),
568        _ => None,
569    }
570}
571
572/// If the register is an AArch64 vector register then return its index.
573fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
574    match reg {
575        InlineAsmReg::AArch64(reg) => reg.vreg_index(),
576        _ => None,
577    }
578}
579
580/// If the register is a Hexagon register pair then return its LLVM double register index.
581/// LLVM uses `d0`, `d1`, ... for Hexagon double registers in inline asm constraints,
582/// not the assembly-printed `r1:0`, `r3:2`, ... format.
583fn hexagon_reg_pair_index(reg: InlineAsmReg) -> Option<u32> {
584    match reg {
585        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r1_0) => Some(0),
586        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r3_2) => Some(1),
587        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r5_4) => Some(2),
588        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r7_6) => Some(3),
589        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r9_8) => Some(4),
590        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r11_10) => Some(5),
591        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r13_12) => Some(6),
592        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r15_14) => Some(7),
593        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r17_16) => Some(8),
594        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r21_20) => Some(10),
595        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r23_22) => Some(11),
596        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r25_24) => Some(12),
597        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r27_26) => Some(13),
598        _ => None,
599    }
600}
601
602/// If the register is a Hexagon HVX vector pair then return its LLVM W-register index.
603/// LLVM uses `w0`, `w1`, ... for Hexagon vector pair registers in inline asm constraints.
604fn hexagon_vreg_pair_index(reg: InlineAsmReg) -> Option<u32> {
605    match reg {
606        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v1_0) => Some(0),
607        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v3_2) => Some(1),
608        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v5_4) => Some(2),
609        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v7_6) => Some(3),
610        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v9_8) => Some(4),
611        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v11_10) => Some(5),
612        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v13_12) => Some(6),
613        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v15_14) => Some(7),
614        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v17_16) => Some(8),
615        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v19_18) => Some(9),
616        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v21_20) => Some(10),
617        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v23_22) => Some(11),
618        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v25_24) => Some(12),
619        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v27_26) => Some(13),
620        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v29_28) => Some(14),
621        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v31_30) => Some(15),
622        _ => None,
623    }
624}
625
626/// Converts a register class to an LLVM constraint code.
627fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
628    use InlineAsmRegClass::*;
629    match reg {
630        // For vector registers LLVM wants the register name to match the type size.
631        InlineAsmRegOrRegClass::Reg(reg) => {
632            if let Some(idx) = xmm_reg_index(reg) {
633                let class = if let Some(layout) = layout {
634                    match layout.size.bytes() {
635                        64 => 'z',
636                        32 => 'y',
637                        _ => 'x',
638                    }
639                } else {
640                    // We use f32 as the type for discarded outputs
641                    'x'
642                };
643                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}mm{1}}}", class, idx))
    })format!("{{{}mm{}}}", class, idx)
644            } else if let Some(idx) = a64_reg_index(reg) {
645                let class = if let Some(layout) = layout {
646                    match layout.size.bytes() {
647                        8 => 'x',
648                        _ => 'w',
649                    }
650                } else {
651                    // We use i32 as the type for discarded outputs
652                    'w'
653                };
654                if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
655                    // LLVM doesn't recognize x30. use lr instead.
656                    "{lr}".to_string()
657                } else {
658                    ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
659                }
660            } else if let Some(idx) = a64_vreg_index(reg) {
661                let class = if let Some(layout) = layout {
662                    match layout.size.bytes() {
663                        16 => 'q',
664                        8 => 'd',
665                        4 => 's',
666                        2 => 'h',
667                        1 => 'd', // We fixup i8 to i8x8
668                        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
669                    }
670                } else {
671                    // We use i64x2 as the type for discarded outputs
672                    'q'
673                };
674                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
675            } else if let Some(idx) = hexagon_reg_pair_index(reg) {
676                // LLVM uses `dN` for Hexagon double registers, not the `rN+1:N` asm syntax.
677                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{d{0}}}", idx))
    })format!("{{d{}}}", idx)
678            } else if let Some(idx) = hexagon_vreg_pair_index(reg) {
679                // LLVM uses `wN` for Hexagon HVX vector pair registers.
680                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{w{0}}}", idx))
    })format!("{{w{}}}", idx)
681            } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
682                // LLVM doesn't recognize r14
683                "{lr}".to_string()
684            } else {
685                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}}}", reg.name()))
    })format!("{{{}}}", reg.name())
686            }
687        }
688        // The constraints can be retrieved from
689        // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
690        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
691            AArch64(AArch64InlineAsmRegClass::reg) => "r",
692            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
693            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
694            AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
695            Arm(ArmInlineAsmRegClass::reg) => "r",
696            Arm(ArmInlineAsmRegClass::sreg)
697            | Arm(ArmInlineAsmRegClass::dreg_low16)
698            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
699            Arm(ArmInlineAsmRegClass::sreg_low16)
700            | Arm(ArmInlineAsmRegClass::dreg_low8)
701            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
702            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
703            Amdgpu(AmdgpuInlineAsmRegClass::Sgpr(_)) => "s",
704            Amdgpu(AmdgpuInlineAsmRegClass::Vgpr(_)) => "v",
705            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
706            Hexagon(HexagonInlineAsmRegClass::reg_pair) => "r",
707            Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
708            Hexagon(HexagonInlineAsmRegClass::vreg) => "v",
709            Hexagon(HexagonInlineAsmRegClass::vreg_pair) => "v",
710            Hexagon(HexagonInlineAsmRegClass::qreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
711            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
712            LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
713            Mips(MipsInlineAsmRegClass::reg) => "r",
714            Mips(MipsInlineAsmRegClass::freg) => "f",
715            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
716            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
717            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
718            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
719            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
720            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
721            PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
722            PowerPC(PowerPCInlineAsmRegClass::vsreg) => "^wa",
723            PowerPC(
724                PowerPCInlineAsmRegClass::cr
725                | PowerPCInlineAsmRegClass::ctr
726                | PowerPCInlineAsmRegClass::lr
727                | PowerPCInlineAsmRegClass::xer
728                | PowerPCInlineAsmRegClass::spe_acc,
729            ) => {
730                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
731            }
732            RiscV(RiscVInlineAsmRegClass::reg) => "r",
733            RiscV(RiscVInlineAsmRegClass::freg) => "f",
734            RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
735            X86(X86InlineAsmRegClass::reg) => "r",
736            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
737            X86(X86InlineAsmRegClass::reg_byte) => "q",
738            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
739            X86(X86InlineAsmRegClass::zmm_reg) => "v",
740            X86(X86InlineAsmRegClass::kreg) => "^Yk",
741            X86(
742                X86InlineAsmRegClass::x87_reg
743                | X86InlineAsmRegClass::mmx_reg
744                | X86InlineAsmRegClass::kreg0
745                | X86InlineAsmRegClass::tmm_reg,
746            ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
747            Xtensa(XtensaInlineAsmRegClass::freg) => "f",
748            Xtensa(XtensaInlineAsmRegClass::reg) => "r",
749            Xtensa(XtensaInlineAsmRegClass::sreg | XtensaInlineAsmRegClass::breg) => {
750                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
751            }
752            Wasm(WasmInlineAsmRegClass::local) => "r",
753            Bpf(BpfInlineAsmRegClass::reg) => "r",
754            Bpf(BpfInlineAsmRegClass::wreg) => "w",
755            Avr(AvrInlineAsmRegClass::reg) => "r",
756            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
757            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
758            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
759            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
760            S390x(S390xInlineAsmRegClass::reg) => "r",
761            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
762            S390x(S390xInlineAsmRegClass::freg) => "f",
763            S390x(S390xInlineAsmRegClass::vreg) => "v",
764            S390x(S390xInlineAsmRegClass::areg) => {
765                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
766            }
767            Sparc(SparcInlineAsmRegClass::reg) => "r",
768            Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
769            Msp430(Msp430InlineAsmRegClass::reg) => "r",
770            M68k(M68kInlineAsmRegClass::reg) => "r",
771            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
772            M68k(M68kInlineAsmRegClass::reg_data) => "d",
773            CSKY(CSKYInlineAsmRegClass::reg) => "r",
774            CSKY(CSKYInlineAsmRegClass::freg) => "f",
775            SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
776            Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
777        }
778        .to_string(),
779    }
780}
781
782/// Converts a modifier into LLVM's equivalent modifier.
783fn modifier_to_llvm(
784    arch: InlineAsmArch,
785    reg: InlineAsmRegClass,
786    modifier: Option<char>,
787) -> Option<char> {
788    use InlineAsmRegClass::*;
789    // The modifiers can be retrieved from
790    // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
791    match reg {
792        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
793        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
794            if modifier == Some('v') {
795                None
796            } else {
797                modifier
798            }
799        }
800        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
801        Arm(ArmInlineAsmRegClass::reg) => None,
802        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
803        Arm(ArmInlineAsmRegClass::dreg)
804        | Arm(ArmInlineAsmRegClass::dreg_low16)
805        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
806        Arm(ArmInlineAsmRegClass::qreg)
807        | Arm(ArmInlineAsmRegClass::qreg_low8)
808        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
809            if modifier.is_none() {
810                Some('q')
811            } else {
812                modifier
813            }
814        }
815        Amdgpu(_) => None,
816        Hexagon(_) => None,
817        LoongArch(_) => None,
818        Mips(_) => None,
819        Nvptx(_) => None,
820        PowerPC(PowerPCInlineAsmRegClass::vsreg) => {
821            // The documentation for the 'x' modifier is missing for llvm, and the gcc
822            // documentation is simply "use this for any vsx argument". It is needed
823            // to ensure the correct vsx register number is used.
824            if modifier.is_none() { Some('x') } else { modifier }
825        }
826        PowerPC(_) => None,
827        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
828        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
829        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
830            None if arch == InlineAsmArch::X86_64 => Some('q'),
831            None => Some('k'),
832            Some('l') => Some('b'),
833            Some('h') => Some('h'),
834            Some('x') => Some('w'),
835            Some('e') => Some('k'),
836            Some('r') => Some('q'),
837            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
838        },
839        X86(X86InlineAsmRegClass::reg_byte) => None,
840        X86(reg @ X86InlineAsmRegClass::xmm_reg)
841        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
842        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
843            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
844            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
845            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
846            (_, Some('x')) => Some('x'),
847            (_, Some('y')) => Some('t'),
848            (_, Some('z')) => Some('g'),
849            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
850        },
851        X86(X86InlineAsmRegClass::kreg) => None,
852        X86(
853            X86InlineAsmRegClass::x87_reg
854            | X86InlineAsmRegClass::mmx_reg
855            | X86InlineAsmRegClass::kreg0
856            | X86InlineAsmRegClass::tmm_reg,
857        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
858        Xtensa(_) => None,
859        Wasm(WasmInlineAsmRegClass::local) => None,
860        Bpf(_) => None,
861        Avr(AvrInlineAsmRegClass::reg_pair)
862        | Avr(AvrInlineAsmRegClass::reg_iw)
863        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
864            Some('h') => Some('B'),
865            Some('l') => Some('A'),
866            _ => None,
867        },
868        Avr(_) => None,
869        S390x(_) => None,
870        Sparc(_) => None,
871        Msp430(_) => None,
872        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
873        M68k(_) => None,
874        CSKY(_) => None,
875        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
876    }
877}
878
879/// Type to use for outputs that are discarded. It doesn't really matter what
880/// the type is, as long as it is valid for the constraint code.
881fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
882    use InlineAsmRegClass::*;
883    match reg {
884        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
885        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
886            cx.type_vector(cx.type_i64(), 2)
887        }
888        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
889        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
890        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
891        Arm(ArmInlineAsmRegClass::dreg)
892        | Arm(ArmInlineAsmRegClass::dreg_low16)
893        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
894        Arm(ArmInlineAsmRegClass::qreg)
895        | Arm(ArmInlineAsmRegClass::qreg_low8)
896        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
897        Amdgpu(_) => cx.type_i32(),
898        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
899        Hexagon(HexagonInlineAsmRegClass::reg_pair) => cx.type_i64(),
900        Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
901        Hexagon(HexagonInlineAsmRegClass::vreg) => {
902            // HVX vector register size depends on the HVX mode.
903            // LLVM's "v" constraint requires the exact vector width.
904            if cx.tcx.sess.unstable_target_features.contains(&sym::hvx_length128b) {
905                cx.type_vector(cx.type_i32(), 32) // 1024-bit for 128B mode
906            } else {
907                cx.type_vector(cx.type_i32(), 16) // 512-bit for 64B mode
908            }
909        }
910        Hexagon(HexagonInlineAsmRegClass::vreg_pair) => {
911            if cx.tcx.sess.unstable_target_features.contains(&sym::hvx_length128b) {
912                cx.type_vector(cx.type_i32(), 64) // 2048-bit for 128B mode
913            } else {
914                cx.type_vector(cx.type_i32(), 32) // 1024-bit for 64B mode
915            }
916        }
917        Hexagon(HexagonInlineAsmRegClass::qreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
918        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
919        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
920        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
921        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
922        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
923        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
924        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
925        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
926        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
927        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
928        PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
929        PowerPC(PowerPCInlineAsmRegClass::vsreg) => cx.type_vector(cx.type_i32(), 4),
930        PowerPC(
931            PowerPCInlineAsmRegClass::cr
932            | PowerPCInlineAsmRegClass::ctr
933            | PowerPCInlineAsmRegClass::lr
934            | PowerPCInlineAsmRegClass::xer
935            | PowerPCInlineAsmRegClass::spe_acc,
936        ) => {
937            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
938        }
939        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
940        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
941        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
942        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
943        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
944        X86(X86InlineAsmRegClass::xmm_reg)
945        | X86(X86InlineAsmRegClass::ymm_reg)
946        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
947        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
948        X86(
949            X86InlineAsmRegClass::x87_reg
950            | X86InlineAsmRegClass::mmx_reg
951            | X86InlineAsmRegClass::kreg0
952            | X86InlineAsmRegClass::tmm_reg,
953        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
954        Xtensa(XtensaInlineAsmRegClass::reg) => cx.type_i32(),
955        Xtensa(XtensaInlineAsmRegClass::freg) => cx.type_f32(),
956        Xtensa(XtensaInlineAsmRegClass::sreg | XtensaInlineAsmRegClass::breg) => {
957            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
958        }
959        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
960        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
961        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
962        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
963        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
964        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
965        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
966        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
967        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
968        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
969        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
970        S390x(S390xInlineAsmRegClass::areg) => {
971            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
972        }
973        Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
974        Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
975        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
976        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
977        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
978        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
979        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
980        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
981        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
982        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
983    }
984}
985
986/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
987/// the equivalent integer type.
988fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
989    let dl = &cx.tcx.data_layout;
990    match scalar.primitive() {
991        Primitive::Int(Integer::I8, _) => cx.type_i8(),
992        Primitive::Int(Integer::I16, _) => cx.type_i16(),
993        Primitive::Int(Integer::I32, _) => cx.type_i32(),
994        Primitive::Int(Integer::I64, _) => cx.type_i64(),
995        Primitive::Float(Float::F16) => cx.type_f16(),
996        Primitive::Float(Float::F32) => cx.type_f32(),
997        Primitive::Float(Float::F64) => cx.type_f64(),
998        Primitive::Float(Float::F128) => cx.type_f128(),
999        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
1000        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
1001        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1002    }
1003}
1004
1005fn any_target_feature_enabled(
1006    cx: &CodegenCx<'_, '_>,
1007    instance: Instance<'_>,
1008    features: &[Symbol],
1009) -> bool {
1010    let enabled = cx.tcx.asm_target_features(instance.def_id());
1011    features.iter().any(|feat| enabled.contains(feat))
1012}
1013
1014/// Fix up an input value to work around LLVM bugs.
1015fn llvm_fixup_input<'ll, 'tcx>(
1016    bx: &mut Builder<'_, 'll, 'tcx>,
1017    mut value: &'ll Value,
1018    reg: InlineAsmRegClass,
1019    layout: &TyAndLayout<'tcx>,
1020    instance: Instance<'_>,
1021) -> &'ll Value {
1022    use InlineAsmRegClass::*;
1023    let dl = &bx.tcx.data_layout;
1024    match (reg, layout.backend_repr) {
1025        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1026            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1027                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
1028                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
1029            } else {
1030                value
1031            }
1032        }
1033        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1034            if s.primitive() != Primitive::Float(Float::F128) =>
1035        {
1036            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
1037            let count = 16 / layout.size.bytes();
1038            let vec_ty = bx.cx.type_vector(elem_ty, count);
1039            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
1040            if let Primitive::Pointer(_) = s.primitive() {
1041                let t = bx.type_from_integer(dl.ptr_sized_integer());
1042                value = bx.ptrtoint(value, t);
1043            }
1044            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
1045        }
1046        (
1047            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1048            BackendRepr::SimdVector { element, count },
1049        ) if layout.size.bytes() == 8 => {
1050            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1051            let vec_ty = bx.cx.type_vector(elem_ty, count);
1052            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
1053            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1054        }
1055        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1056            if s.primitive() == Primitive::Float(Float::F64) =>
1057        {
1058            bx.bitcast(value, bx.cx.type_i64())
1059        }
1060        (
1061            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1062            BackendRepr::SimdVector { .. },
1063        ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
1064        (
1065            X86(
1066                X86InlineAsmRegClass::xmm_reg
1067                | X86InlineAsmRegClass::ymm_reg
1068                | X86InlineAsmRegClass::zmm_reg,
1069            ),
1070            BackendRepr::Scalar(s),
1071        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1072            && s.primitive() == Primitive::Float(Float::F128) =>
1073        {
1074            bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
1075        }
1076        (
1077            X86(
1078                X86InlineAsmRegClass::xmm_reg
1079                | X86InlineAsmRegClass::ymm_reg
1080                | X86InlineAsmRegClass::zmm_reg,
1081            ),
1082            BackendRepr::Scalar(s),
1083        ) if s.primitive() == Primitive::Float(Float::F16) => {
1084            let value = bx.insert_element(
1085                bx.const_undef(bx.type_vector(bx.type_f16(), 8)),
1086                value,
1087                bx.const_usize(0),
1088            );
1089            bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
1090        }
1091        (
1092            X86(
1093                X86InlineAsmRegClass::xmm_reg
1094                | X86InlineAsmRegClass::ymm_reg
1095                | X86InlineAsmRegClass::zmm_reg,
1096            ),
1097            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1098        ) if element.primitive() == Primitive::Float(Float::F16) => {
1099            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1100        }
1101        (
1102            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1103            BackendRepr::Scalar(s),
1104        ) => {
1105            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1106                bx.bitcast(value, bx.cx.type_f32())
1107            } else {
1108                value
1109            }
1110        }
1111        (
1112            Arm(
1113                ArmInlineAsmRegClass::dreg
1114                | ArmInlineAsmRegClass::dreg_low8
1115                | ArmInlineAsmRegClass::dreg_low16,
1116            ),
1117            BackendRepr::Scalar(s),
1118        ) => {
1119            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1120                bx.bitcast(value, bx.cx.type_f64())
1121            } else {
1122                value
1123            }
1124        }
1125        (
1126            Arm(
1127                ArmInlineAsmRegClass::dreg
1128                | ArmInlineAsmRegClass::dreg_low8
1129                | ArmInlineAsmRegClass::dreg_low16
1130                | ArmInlineAsmRegClass::qreg
1131                | ArmInlineAsmRegClass::qreg_low4
1132                | ArmInlineAsmRegClass::qreg_low8,
1133            ),
1134            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1135        ) if element.primitive() == Primitive::Float(Float::F16) => {
1136            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1137        }
1138        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1139            if s.primitive() == Primitive::Float(Float::F16) =>
1140        {
1141            // Smaller floats are always "NaN-boxed" inside larger floats on LoongArch.
1142            let value = bx.bitcast(value, bx.type_i16());
1143            let value = bx.zext(value, bx.type_i32());
1144            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1145            bx.bitcast(value, bx.type_f32())
1146        }
1147        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1148            match s.primitive() {
1149                // MIPS only supports register-length arithmetics.
1150                Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
1151                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()),
1152                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()),
1153                _ => value,
1154            }
1155        }
1156        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1157            if s.primitive() == Primitive::Float(Float::F16)
1158                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1159        {
1160            // Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1161            let value = bx.bitcast(value, bx.type_i16());
1162            let value = bx.zext(value, bx.type_i32());
1163            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1164            bx.bitcast(value, bx.type_f32())
1165        }
1166        (
1167            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1168            BackendRepr::Scalar(s),
1169        ) if s.primitive() == Primitive::Float(Float::F32) => {
1170            let value = bx.insert_element(
1171                bx.const_undef(bx.type_vector(bx.type_f32(), 4)),
1172                value,
1173                bx.const_usize(0),
1174            );
1175            bx.bitcast(value, bx.type_vector(bx.type_f32(), 4))
1176        }
1177        (
1178            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1179            BackendRepr::Scalar(s),
1180        ) if s.primitive() == Primitive::Float(Float::F64) => {
1181            let value = bx.insert_element(
1182                bx.const_undef(bx.type_vector(bx.type_f64(), 2)),
1183                value,
1184                bx.const_usize(0),
1185            );
1186            bx.bitcast(value, bx.type_vector(bx.type_f64(), 2))
1187        }
1188        _ => value,
1189    }
1190}
1191
1192/// Fix up an output value to work around LLVM bugs.
1193fn llvm_fixup_output<'ll, 'tcx>(
1194    bx: &mut Builder<'_, 'll, 'tcx>,
1195    mut value: &'ll Value,
1196    reg: InlineAsmRegClass,
1197    layout: &TyAndLayout<'tcx>,
1198    instance: Instance<'_>,
1199) -> &'ll Value {
1200    use InlineAsmRegClass::*;
1201    match (reg, layout.backend_repr) {
1202        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1203            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1204                bx.extract_element(value, bx.const_i32(0))
1205            } else {
1206                value
1207            }
1208        }
1209        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1210            if s.primitive() != Primitive::Float(Float::F128) =>
1211        {
1212            value = bx.extract_element(value, bx.const_i32(0));
1213            if let Primitive::Pointer(_) = s.primitive() {
1214                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
1215            }
1216            value
1217        }
1218        (
1219            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1220            BackendRepr::SimdVector { element, count },
1221        ) if layout.size.bytes() == 8 => {
1222            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1223            let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1224            let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1225            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1226        }
1227        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1228            if s.primitive() == Primitive::Float(Float::F64) =>
1229        {
1230            bx.bitcast(value, bx.cx.type_f64())
1231        }
1232        (
1233            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1234            BackendRepr::SimdVector { .. },
1235        ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1236        (
1237            X86(
1238                X86InlineAsmRegClass::xmm_reg
1239                | X86InlineAsmRegClass::ymm_reg
1240                | X86InlineAsmRegClass::zmm_reg,
1241            ),
1242            BackendRepr::Scalar(s),
1243        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1244            && s.primitive() == Primitive::Float(Float::F128) =>
1245        {
1246            bx.bitcast(value, bx.type_f128())
1247        }
1248        (
1249            X86(
1250                X86InlineAsmRegClass::xmm_reg
1251                | X86InlineAsmRegClass::ymm_reg
1252                | X86InlineAsmRegClass::zmm_reg,
1253            ),
1254            BackendRepr::Scalar(s),
1255        ) if s.primitive() == Primitive::Float(Float::F16) => {
1256            let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8));
1257            bx.extract_element(value, bx.const_usize(0))
1258        }
1259        (
1260            X86(
1261                X86InlineAsmRegClass::xmm_reg
1262                | X86InlineAsmRegClass::ymm_reg
1263                | X86InlineAsmRegClass::zmm_reg,
1264            ),
1265            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1266        ) if element.primitive() == Primitive::Float(Float::F16) => {
1267            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1268        }
1269        (
1270            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1271            BackendRepr::Scalar(s),
1272        ) => {
1273            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1274                bx.bitcast(value, bx.cx.type_i32())
1275            } else {
1276                value
1277            }
1278        }
1279        (
1280            Arm(
1281                ArmInlineAsmRegClass::dreg
1282                | ArmInlineAsmRegClass::dreg_low8
1283                | ArmInlineAsmRegClass::dreg_low16,
1284            ),
1285            BackendRepr::Scalar(s),
1286        ) => {
1287            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1288                bx.bitcast(value, bx.cx.type_i64())
1289            } else {
1290                value
1291            }
1292        }
1293        (
1294            Arm(
1295                ArmInlineAsmRegClass::dreg
1296                | ArmInlineAsmRegClass::dreg_low8
1297                | ArmInlineAsmRegClass::dreg_low16
1298                | ArmInlineAsmRegClass::qreg
1299                | ArmInlineAsmRegClass::qreg_low4
1300                | ArmInlineAsmRegClass::qreg_low8,
1301            ),
1302            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1303        ) if element.primitive() == Primitive::Float(Float::F16) => {
1304            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1305        }
1306        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1307            if s.primitive() == Primitive::Float(Float::F16) =>
1308        {
1309            let value = bx.bitcast(value, bx.type_i32());
1310            let value = bx.trunc(value, bx.type_i16());
1311            bx.bitcast(value, bx.type_f16())
1312        }
1313        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1314            match s.primitive() {
1315                // MIPS only supports register-length arithmetics.
1316                Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1317                Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1318                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()),
1319                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()),
1320                _ => value,
1321            }
1322        }
1323        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1324            if s.primitive() == Primitive::Float(Float::F16)
1325                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1326        {
1327            let value = bx.bitcast(value, bx.type_i32());
1328            let value = bx.trunc(value, bx.type_i16());
1329            bx.bitcast(value, bx.type_f16())
1330        }
1331        (
1332            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1333            BackendRepr::Scalar(s),
1334        ) if s.primitive() == Primitive::Float(Float::F32) => {
1335            let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4));
1336            bx.extract_element(value, bx.const_usize(0))
1337        }
1338        (
1339            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1340            BackendRepr::Scalar(s),
1341        ) if s.primitive() == Primitive::Float(Float::F64) => {
1342            let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2));
1343            bx.extract_element(value, bx.const_usize(0))
1344        }
1345        _ => value,
1346    }
1347}
1348
1349/// Output type to use for llvm_fixup_output.
1350fn llvm_fixup_output_type<'ll, 'tcx>(
1351    cx: &CodegenCx<'ll, 'tcx>,
1352    reg: InlineAsmRegClass,
1353    layout: &TyAndLayout<'tcx>,
1354    instance: Instance<'_>,
1355) -> &'ll Type {
1356    use InlineAsmRegClass::*;
1357    match (reg, layout.backend_repr) {
1358        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1359            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1360                cx.type_vector(cx.type_i8(), 8)
1361            } else {
1362                layout.llvm_type(cx)
1363            }
1364        }
1365        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1366            if s.primitive() != Primitive::Float(Float::F128) =>
1367        {
1368            let elem_ty = llvm_asm_scalar_type(cx, s);
1369            let count = 16 / layout.size.bytes();
1370            cx.type_vector(elem_ty, count)
1371        }
1372        (
1373            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1374            BackendRepr::SimdVector { element, count },
1375        ) if layout.size.bytes() == 8 => {
1376            let elem_ty = llvm_asm_scalar_type(cx, element);
1377            cx.type_vector(elem_ty, count * 2)
1378        }
1379        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1380            if s.primitive() == Primitive::Float(Float::F64) =>
1381        {
1382            cx.type_i64()
1383        }
1384        (
1385            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1386            BackendRepr::SimdVector { .. },
1387        ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1388        (
1389            X86(
1390                X86InlineAsmRegClass::xmm_reg
1391                | X86InlineAsmRegClass::ymm_reg
1392                | X86InlineAsmRegClass::zmm_reg,
1393            ),
1394            BackendRepr::Scalar(s),
1395        ) if cx.sess().asm_arch == Some(InlineAsmArch::X86)
1396            && s.primitive() == Primitive::Float(Float::F128) =>
1397        {
1398            cx.type_vector(cx.type_i32(), 4)
1399        }
1400        (
1401            X86(
1402                X86InlineAsmRegClass::xmm_reg
1403                | X86InlineAsmRegClass::ymm_reg
1404                | X86InlineAsmRegClass::zmm_reg,
1405            ),
1406            BackendRepr::Scalar(s),
1407        ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
1408        (
1409            X86(
1410                X86InlineAsmRegClass::xmm_reg
1411                | X86InlineAsmRegClass::ymm_reg
1412                | X86InlineAsmRegClass::zmm_reg,
1413            ),
1414            BackendRepr::SimdVector { element, count: count @ (8 | 16) },
1415        ) if element.primitive() == Primitive::Float(Float::F16) => {
1416            cx.type_vector(cx.type_i16(), count)
1417        }
1418        (
1419            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1420            BackendRepr::Scalar(s),
1421        ) => {
1422            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1423                cx.type_f32()
1424            } else {
1425                layout.llvm_type(cx)
1426            }
1427        }
1428        (
1429            Arm(
1430                ArmInlineAsmRegClass::dreg
1431                | ArmInlineAsmRegClass::dreg_low8
1432                | ArmInlineAsmRegClass::dreg_low16,
1433            ),
1434            BackendRepr::Scalar(s),
1435        ) => {
1436            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1437                cx.type_f64()
1438            } else {
1439                layout.llvm_type(cx)
1440            }
1441        }
1442        (
1443            Arm(
1444                ArmInlineAsmRegClass::dreg
1445                | ArmInlineAsmRegClass::dreg_low8
1446                | ArmInlineAsmRegClass::dreg_low16
1447                | ArmInlineAsmRegClass::qreg
1448                | ArmInlineAsmRegClass::qreg_low4
1449                | ArmInlineAsmRegClass::qreg_low8,
1450            ),
1451            BackendRepr::SimdVector { element, count: count @ (4 | 8) },
1452        ) if element.primitive() == Primitive::Float(Float::F16) => {
1453            cx.type_vector(cx.type_i16(), count)
1454        }
1455        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1456            if s.primitive() == Primitive::Float(Float::F16) =>
1457        {
1458            cx.type_f32()
1459        }
1460        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1461            match s.primitive() {
1462                // MIPS only supports register-length arithmetics.
1463                Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1464                Primitive::Float(Float::F32) => cx.type_i32(),
1465                Primitive::Float(Float::F64) => cx.type_i64(),
1466                _ => layout.llvm_type(cx),
1467            }
1468        }
1469        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1470            if s.primitive() == Primitive::Float(Float::F16)
1471                && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1472        {
1473            cx.type_f32()
1474        }
1475        (
1476            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1477            BackendRepr::Scalar(s),
1478        ) if s.primitive() == Primitive::Float(Float::F32) => cx.type_vector(cx.type_f32(), 4),
1479        (
1480            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1481            BackendRepr::Scalar(s),
1482        ) if s.primitive() == Primitive::Float(Float::F64) => cx.type_vector(cx.type_f64(), 2),
1483        _ => layout.llvm_type(cx),
1484    }
1485}