core/stdarch/crates/core_arch/src/
simd.rs

1//! Internal `#[repr(simd)]` types
2
3#![allow(non_camel_case_types)]
4
5#[inline(always)]
6#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
7pub(crate) const unsafe fn simd_imax<T: Copy>(a: T, b: T) -> T {
8    let mask: T = crate::intrinsics::simd::simd_gt(a, b);
9    crate::intrinsics::simd::simd_select(mask, a, b)
10}
11
12#[inline(always)]
13#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
14pub(crate) const unsafe fn simd_imin<T: Copy>(a: T, b: T) -> T {
15    let mask: T = crate::intrinsics::simd::simd_lt(a, b);
16    crate::intrinsics::simd::simd_select(mask, a, b)
17}
18
19/// SAFETY: All bits patterns must be valid
20pub(crate) unsafe trait SimdElement:
21    Copy + const PartialEq + crate::fmt::Debug
22{
23    // SAFETY: all bits patterns of types implementing this trait must be valid
24    const ZERO: Self = unsafe { crate::mem::zeroed() };
25}
26
27unsafe impl SimdElement for u8 {}
28unsafe impl SimdElement for u16 {}
29unsafe impl SimdElement for u32 {}
30unsafe impl SimdElement for u64 {}
31unsafe impl SimdElement for u128 {}
32
33unsafe impl SimdElement for i8 {}
34unsafe impl SimdElement for i16 {}
35unsafe impl SimdElement for i32 {}
36unsafe impl SimdElement for i64 {}
37unsafe impl SimdElement for i128 {}
38
39unsafe impl SimdElement for f16 {}
40unsafe impl SimdElement for f32 {}
41unsafe impl SimdElement for f64 {}
42
43#[repr(simd)]
44#[derive(Copy)]
45pub(crate) struct Simd<T: SimdElement, const N: usize>([T; N]);
46
47impl<T: SimdElement, const N: usize> Simd<T, N> {
48    /// A value of this type where all elements are zeroed out.
49    pub(crate) const ZERO: Self = Self::splat(T::ZERO);
50
51    #[inline(always)]
52    pub(crate) const fn from_array(elements: [T; N]) -> Self {
53        Self(elements)
54    }
55
56    #[inline]
57    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
58    pub(crate) const fn splat(value: T) -> Self {
59        unsafe { crate::intrinsics::simd::simd_splat(value) }
60    }
61
62    /// Extract the element at position `index`. Note that `index` is not a constant so this
63    /// operation is not efficient on most platforms. Use for testing only.
64    #[inline]
65    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
66    pub(crate) const fn extract_dyn(&self, index: usize) -> T {
67        assert!(index < N);
68        // SAFETY: self is a vector, T its element type.
69        unsafe { crate::intrinsics::simd::simd_extract_dyn(*self, index as u32) }
70    }
71
72    #[inline]
73    pub(crate) const fn as_array(&self) -> &[T; N] {
74        let simd_ptr: *const Self = self;
75        let array_ptr: *const [T; N] = simd_ptr.cast();
76        // SAFETY: We can always read the prefix of a simd type as an array.
77        // There might be more padding afterwards for some widths, but
78        // that's not a problem for reading less than that.
79        unsafe { &*array_ptr }
80    }
81}
82
83// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::Simd is banned by MCP#838"
84impl<T: SimdElement, const N: usize> Clone for Simd<T, N> {
85    #[inline]
86    fn clone(&self) -> Self {
87        *self
88    }
89}
90
91#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
92#[rustfmt::skip] // FIXME: https://github.com/rust-lang/stdarch/pull/2133#issuecomment-4524350350
93const impl<T: SimdElement, const N: usize> crate::cmp::PartialEq for Simd<T, N> {
94    #[inline]
95    fn eq(&self, other: &Self) -> bool {
96        self.as_array() == other.as_array()
97    }
98}
99
100impl<T: SimdElement, const N: usize> crate::fmt::Debug for Simd<T, N> {
101    #[inline]
102    fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result {
103        debug_simd_finish(f, "Simd", self.as_array())
104    }
105}
106
107impl<T: SimdElement> Simd<T, 1> {
108    #[inline]
109    pub(crate) const fn new(x0: T) -> Self {
110        Self([x0])
111    }
112}
113
114impl<T: SimdElement> Simd<T, 2> {
115    #[inline]
116    pub(crate) const fn new(x0: T, x1: T) -> Self {
117        Self([x0, x1])
118    }
119}
120
121impl<T: SimdElement> Simd<T, 4> {
122    #[inline]
123    pub(crate) const fn new(x0: T, x1: T, x2: T, x3: T) -> Self {
124        Self([x0, x1, x2, x3])
125    }
126}
127
128impl<T: SimdElement> Simd<T, 8> {
129    #[inline]
130    pub(crate) const fn new(x0: T, x1: T, x2: T, x3: T, x4: T, x5: T, x6: T, x7: T) -> Self {
131        Self([x0, x1, x2, x3, x4, x5, x6, x7])
132    }
133}
134
135impl<T: SimdElement> Simd<T, 16> {
136    #[inline]
137    pub(crate) const fn new(
138        x0: T,
139        x1: T,
140        x2: T,
141        x3: T,
142        x4: T,
143        x5: T,
144        x6: T,
145        x7: T,
146        x8: T,
147        x9: T,
148        x10: T,
149        x11: T,
150        x12: T,
151        x13: T,
152        x14: T,
153        x15: T,
154    ) -> Self {
155        Self([
156            x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15,
157        ])
158    }
159}
160
161impl<T: SimdElement> Simd<T, 32> {
162    #[inline]
163    pub(crate) const fn new(
164        x0: T,
165        x1: T,
166        x2: T,
167        x3: T,
168        x4: T,
169        x5: T,
170        x6: T,
171        x7: T,
172        x8: T,
173        x9: T,
174        x10: T,
175        x11: T,
176        x12: T,
177        x13: T,
178        x14: T,
179        x15: T,
180        x16: T,
181        x17: T,
182        x18: T,
183        x19: T,
184        x20: T,
185        x21: T,
186        x22: T,
187        x23: T,
188        x24: T,
189        x25: T,
190        x26: T,
191        x27: T,
192        x28: T,
193        x29: T,
194        x30: T,
195        x31: T,
196    ) -> Self {
197        Self([
198            x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x18,
199            x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, x30, x31,
200        ])
201    }
202}
203
204impl<const N: usize> Simd<f16, N> {
205    #[inline]
206    pub(crate) const fn to_bits(self) -> Simd<u16, N> {
207        assert!(size_of::<Self>() == size_of::<Simd<u16, N>>());
208        unsafe { crate::mem::transmute_copy(&self) }
209    }
210
211    #[inline]
212    pub(crate) const fn from_bits(bits: Simd<u16, N>) -> Self {
213        assert!(size_of::<Self>() == size_of::<Simd<u16, N>>());
214        unsafe { crate::mem::transmute_copy(&bits) }
215    }
216}
217
218impl<const N: usize> Simd<f32, N> {
219    #[inline]
220    pub(crate) const fn to_bits(self) -> Simd<u32, N> {
221        assert!(size_of::<Self>() == size_of::<Simd<u32, N>>());
222        unsafe { crate::mem::transmute_copy(&self) }
223    }
224
225    #[inline]
226    pub(crate) const fn from_bits(bits: Simd<u32, N>) -> Self {
227        assert!(size_of::<Self>() == size_of::<Simd<u32, N>>());
228        unsafe { crate::mem::transmute_copy(&bits) }
229    }
230}
231
232impl<const N: usize> Simd<f64, N> {
233    #[inline]
234    pub(crate) const fn to_bits(self) -> Simd<u64, N> {
235        assert!(size_of::<Self>() == size_of::<Simd<u64, N>>());
236        unsafe { crate::mem::transmute_copy(&self) }
237    }
238
239    #[inline]
240    pub(crate) const fn from_bits(bits: Simd<u64, N>) -> Self {
241        assert!(size_of::<Self>() == size_of::<Simd<u64, N>>());
242        unsafe { crate::mem::transmute_copy(&bits) }
243    }
244}
245
246#[repr(simd)]
247#[derive(Copy)]
248pub(crate) struct SimdM<T: SimdElement, const N: usize>([T; N]);
249
250impl<T: SimdElement, const N: usize> SimdM<T, N> {
251    #[inline(always)]
252    const fn bool_to_internal(x: bool) -> T {
253        // SAFETY: `T` implements `SimdElement`, so all bit patterns are valid.
254        let ones = const {
255            // Ideally, this would be `transmute([0xFFu8; size_of::<T>()])`, but
256            // `size_of::<T>()` is not allowed to use a generic parameter there.
257            let mut r = crate::mem::MaybeUninit::<T>::uninit();
258            let mut i = 0;
259            while i < crate::mem::size_of::<T>() {
260                r.as_bytes_mut()[i] = crate::mem::MaybeUninit::new(0xFF);
261                i += 1;
262            }
263            unsafe { r.assume_init() }
264        };
265        [T::ZERO, ones][x as usize]
266    }
267
268    #[inline]
269    pub(crate) const fn from_array(elements: [bool; N]) -> Self {
270        let mut internal = [T::ZERO; N];
271        let mut i = 0;
272        while i < N {
273            internal[i] = Self::bool_to_internal(elements[i]);
274            i += 1;
275        }
276        Self(internal)
277    }
278
279    #[inline]
280    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
281    pub(crate) const fn splat(value: bool) -> Self {
282        unsafe { crate::intrinsics::simd::simd_splat(Self::bool_to_internal(value)) }
283    }
284
285    #[inline]
286    pub(crate) const fn as_array(&self) -> &[T; N] {
287        let simd_ptr: *const Self = self;
288        let array_ptr: *const [T; N] = simd_ptr.cast();
289        // SAFETY: We can always read the prefix of a simd type as an array.
290        // There might be more padding afterwards for some widths, but
291        // that's not a problem for reading less than that.
292        unsafe { &*array_ptr }
293    }
294}
295
296// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::SimdM is banned by MCP#838"
297impl<T: SimdElement, const N: usize> Clone for SimdM<T, N> {
298    #[inline]
299    fn clone(&self) -> Self {
300        *self
301    }
302}
303
304#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
305#[rustfmt::skip] // FIXME: https://github.com/rust-lang/stdarch/pull/2133#issuecomment-4524350350
306const impl<T: SimdElement, const N: usize> crate::cmp::PartialEq for SimdM<T, N> {
307    #[inline]
308    fn eq(&self, other: &Self) -> bool {
309        self.as_array() == other.as_array()
310    }
311}
312
313impl<T: SimdElement, const N: usize> crate::fmt::Debug for SimdM<T, N> {
314    #[inline]
315    fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result {
316        debug_simd_finish(f, "SimdM", self.as_array())
317    }
318}
319
320// 16-bit wide types:
321
322pub(crate) type u8x2 = Simd<u8, 2>;
323pub(crate) type i8x2 = Simd<i8, 2>;
324
325// 32-bit wide types:
326
327pub(crate) type u8x4 = Simd<u8, 4>;
328pub(crate) type u16x2 = Simd<u16, 2>;
329
330pub(crate) type i8x4 = Simd<i8, 4>;
331pub(crate) type i16x2 = Simd<i16, 2>;
332
333// 64-bit wide types:
334
335pub(crate) type u8x8 = Simd<u8, 8>;
336pub(crate) type u16x4 = Simd<u16, 4>;
337pub(crate) type u32x2 = Simd<u32, 2>;
338pub(crate) type u64x1 = Simd<u64, 1>;
339
340pub(crate) type i8x8 = Simd<i8, 8>;
341pub(crate) type i16x4 = Simd<i16, 4>;
342pub(crate) type i32x2 = Simd<i32, 2>;
343pub(crate) type i64x1 = Simd<i64, 1>;
344
345pub(crate) type f16x4 = Simd<f16, 4>;
346pub(crate) type f32x2 = Simd<f32, 2>;
347pub(crate) type f64x1 = Simd<f64, 1>;
348
349// 128-bit wide types:
350
351pub(crate) type u8x16 = Simd<u8, 16>;
352pub(crate) type u16x8 = Simd<u16, 8>;
353pub(crate) type u32x4 = Simd<u32, 4>;
354pub(crate) type u64x2 = Simd<u64, 2>;
355
356pub(crate) type i8x16 = Simd<i8, 16>;
357pub(crate) type i16x8 = Simd<i16, 8>;
358pub(crate) type i32x4 = Simd<i32, 4>;
359pub(crate) type i64x2 = Simd<i64, 2>;
360
361pub(crate) type f16x8 = Simd<f16, 8>;
362pub(crate) type f32x4 = Simd<f32, 4>;
363pub(crate) type f64x2 = Simd<f64, 2>;
364
365pub(crate) type m8x16 = SimdM<i8, 16>;
366pub(crate) type m16x8 = SimdM<i16, 8>;
367pub(crate) type m32x4 = SimdM<i32, 4>;
368pub(crate) type m64x2 = SimdM<i64, 2>;
369
370// 256-bit wide types:
371
372pub(crate) type u8x32 = Simd<u8, 32>;
373pub(crate) type u16x16 = Simd<u16, 16>;
374pub(crate) type u32x8 = Simd<u32, 8>;
375pub(crate) type u64x4 = Simd<u64, 4>;
376pub(crate) type u128x2 = Simd<u128, 2>;
377
378pub(crate) type i8x32 = Simd<i8, 32>;
379pub(crate) type i16x16 = Simd<i16, 16>;
380pub(crate) type i32x8 = Simd<i32, 8>;
381pub(crate) type i64x4 = Simd<i64, 4>;
382pub(crate) type i128x2 = Simd<i128, 2>;
383
384pub(crate) type f16x16 = Simd<f16, 16>;
385pub(crate) type f32x8 = Simd<f32, 8>;
386pub(crate) type f64x4 = Simd<f64, 4>;
387
388pub(crate) type m8x32 = SimdM<i8, 32>;
389pub(crate) type m16x16 = SimdM<i16, 16>;
390pub(crate) type m32x8 = SimdM<i32, 8>;
391
392// 512-bit wide types:
393
394pub(crate) type u8x64 = Simd<u8, 64>;
395pub(crate) type u16x32 = Simd<u16, 32>;
396pub(crate) type u32x16 = Simd<u32, 16>;
397pub(crate) type u64x8 = Simd<u64, 8>;
398pub(crate) type u128x4 = Simd<u128, 4>;
399
400pub(crate) type i8x64 = Simd<i8, 64>;
401pub(crate) type i16x32 = Simd<i16, 32>;
402pub(crate) type i32x16 = Simd<i32, 16>;
403pub(crate) type i64x8 = Simd<i64, 8>;
404pub(crate) type i128x4 = Simd<i128, 4>;
405
406pub(crate) type f16x32 = Simd<f16, 32>;
407pub(crate) type f32x16 = Simd<f32, 16>;
408pub(crate) type f64x8 = Simd<f64, 8>;
409
410// 1024-bit wide types:
411
412pub(crate) type u16x64 = Simd<u16, 64>;
413pub(crate) type u32x32 = Simd<u32, 32>;
414
415pub(crate) type i32x32 = Simd<i32, 32>;
416
417/// Used to continue `Debug`ging SIMD types as `MySimd(1, 2, 3, 4)`, as they
418/// were before moving to array-based simd.
419#[inline]
420pub(crate) fn debug_simd_finish<T: crate::fmt::Debug, const N: usize>(
421    formatter: &mut crate::fmt::Formatter<'_>,
422    type_name: &str,
423    array: &[T; N],
424) -> crate::fmt::Result {
425    crate::fmt::Formatter::debug_tuple_fields_finish(
426        formatter,
427        type_name,
428        &crate::array::from_fn::<&dyn crate::fmt::Debug, N, _>(|i| &array[i]),
429    )
430}
core/stdarch/crates/core_arch/src/simd.rs

core/stdarch/crates/core_arch/src/
simd.rs
