core/stdarch/crates/core_arch/src/
simd.rs

1//! Internal `#[repr(simd)]` types
2
3#![allow(non_camel_case_types)]
4
5#[inline(always)]
6#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
7pub(crate) const unsafe fn simd_imax<T: Copy>(a: T, b: T) -> T {
8    let mask: T = crate::intrinsics::simd::simd_gt(a, b);
9    crate::intrinsics::simd::simd_select(mask, a, b)
10}
11
12#[inline(always)]
13#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
14pub(crate) const unsafe fn simd_imin<T: Copy>(a: T, b: T) -> T {
15    let mask: T = crate::intrinsics::simd::simd_lt(a, b);
16    crate::intrinsics::simd::simd_select(mask, a, b)
17}
18
19/// SAFETY: All bits patterns must be valid
20pub(crate) unsafe trait SimdElement:
21    Copy + const PartialEq + crate::fmt::Debug
22{
23    // SAFETY: all bits patterns of types implementing this trait must be valid
24    const ZERO: Self = unsafe { crate::mem::zeroed() };
25}
26
27unsafe impl SimdElement for u8 {}
28unsafe impl SimdElement for u16 {}
29unsafe impl SimdElement for u32 {}
30unsafe impl SimdElement for u64 {}
31
32unsafe impl SimdElement for i8 {}
33unsafe impl SimdElement for i16 {}
34unsafe impl SimdElement for i32 {}
35unsafe impl SimdElement for i64 {}
36
37unsafe impl SimdElement for f16 {}
38unsafe impl SimdElement for f32 {}
39unsafe impl SimdElement for f64 {}
40
41#[repr(simd)]
42#[derive(Copy)]
43pub(crate) struct Simd<T: SimdElement, const N: usize>([T; N]);
44
45impl<T: SimdElement, const N: usize> Simd<T, N> {
46    /// A value of this type where all elements are zeroed out.
47    pub(crate) const ZERO: Self = Self::splat(T::ZERO);
48
49    #[inline(always)]
50    pub(crate) const fn from_array(elements: [T; N]) -> Self {
51        Self(elements)
52    }
53
54    #[inline]
55    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
56    pub(crate) const fn splat(value: T) -> Self {
57        unsafe { crate::intrinsics::simd::simd_splat(value) }
58    }
59
60    /// Extract the element at position `index`. Note that `index` is not a constant so this
61    /// operation is not efficient on most platforms. Use for testing only.
62    #[inline]
63    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
64    pub(crate) const fn extract_dyn(&self, index: usize) -> T {
65        assert!(index < N);
66        // SAFETY: self is a vector, T its element type.
67        unsafe { crate::intrinsics::simd::simd_extract_dyn(*self, index as u32) }
68    }
69
70    #[inline]
71    pub(crate) const fn as_array(&self) -> &[T; N] {
72        let simd_ptr: *const Self = self;
73        let array_ptr: *const [T; N] = simd_ptr.cast();
74        // SAFETY: We can always read the prefix of a simd type as an array.
75        // There might be more padding afterwards for some widths, but
76        // that's not a problem for reading less than that.
77        unsafe { &*array_ptr }
78    }
79}
80
81// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::Simd is banned by MCP#838"
82impl<T: SimdElement, const N: usize> Clone for Simd<T, N> {
83    #[inline]
84    fn clone(&self) -> Self {
85        *self
86    }
87}
88
89#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
90#[rustfmt::skip] // FIXME: https://github.com/rust-lang/stdarch/pull/2133#issuecomment-4524350350
91impl<T: SimdElement, const N: usize> const crate::cmp::PartialEq for Simd<T, N> {
92    #[inline]
93    fn eq(&self, other: &Self) -> bool {
94        self.as_array() == other.as_array()
95    }
96}
97
98impl<T: SimdElement, const N: usize> crate::fmt::Debug for Simd<T, N> {
99    #[inline]
100    fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result {
101        debug_simd_finish(f, "Simd", self.as_array())
102    }
103}
104
105impl<T: SimdElement> Simd<T, 1> {
106    #[inline]
107    pub(crate) const fn new(x0: T) -> Self {
108        Self([x0])
109    }
110}
111
112impl<T: SimdElement> Simd<T, 2> {
113    #[inline]
114    pub(crate) const fn new(x0: T, x1: T) -> Self {
115        Self([x0, x1])
116    }
117}
118
119impl<T: SimdElement> Simd<T, 4> {
120    #[inline]
121    pub(crate) const fn new(x0: T, x1: T, x2: T, x3: T) -> Self {
122        Self([x0, x1, x2, x3])
123    }
124}
125
126impl<T: SimdElement> Simd<T, 8> {
127    #[inline]
128    pub(crate) const fn new(x0: T, x1: T, x2: T, x3: T, x4: T, x5: T, x6: T, x7: T) -> Self {
129        Self([x0, x1, x2, x3, x4, x5, x6, x7])
130    }
131}
132
133impl<T: SimdElement> Simd<T, 16> {
134    #[inline]
135    pub(crate) const fn new(
136        x0: T,
137        x1: T,
138        x2: T,
139        x3: T,
140        x4: T,
141        x5: T,
142        x6: T,
143        x7: T,
144        x8: T,
145        x9: T,
146        x10: T,
147        x11: T,
148        x12: T,
149        x13: T,
150        x14: T,
151        x15: T,
152    ) -> Self {
153        Self([
154            x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15,
155        ])
156    }
157}
158
159impl<T: SimdElement> Simd<T, 32> {
160    #[inline]
161    pub(crate) const fn new(
162        x0: T,
163        x1: T,
164        x2: T,
165        x3: T,
166        x4: T,
167        x5: T,
168        x6: T,
169        x7: T,
170        x8: T,
171        x9: T,
172        x10: T,
173        x11: T,
174        x12: T,
175        x13: T,
176        x14: T,
177        x15: T,
178        x16: T,
179        x17: T,
180        x18: T,
181        x19: T,
182        x20: T,
183        x21: T,
184        x22: T,
185        x23: T,
186        x24: T,
187        x25: T,
188        x26: T,
189        x27: T,
190        x28: T,
191        x29: T,
192        x30: T,
193        x31: T,
194    ) -> Self {
195        Self([
196            x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x18,
197            x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, x30, x31,
198        ])
199    }
200}
201
202impl<const N: usize> Simd<f16, N> {
203    #[inline]
204    pub(crate) const fn to_bits(self) -> Simd<u16, N> {
205        assert!(size_of::<Self>() == size_of::<Simd<u16, N>>());
206        unsafe { crate::mem::transmute_copy(&self) }
207    }
208
209    #[inline]
210    pub(crate) const fn from_bits(bits: Simd<u16, N>) -> Self {
211        assert!(size_of::<Self>() == size_of::<Simd<u16, N>>());
212        unsafe { crate::mem::transmute_copy(&bits) }
213    }
214}
215
216impl<const N: usize> Simd<f32, N> {
217    #[inline]
218    pub(crate) const fn to_bits(self) -> Simd<u32, N> {
219        assert!(size_of::<Self>() == size_of::<Simd<u32, N>>());
220        unsafe { crate::mem::transmute_copy(&self) }
221    }
222
223    #[inline]
224    pub(crate) const fn from_bits(bits: Simd<u32, N>) -> Self {
225        assert!(size_of::<Self>() == size_of::<Simd<u32, N>>());
226        unsafe { crate::mem::transmute_copy(&bits) }
227    }
228}
229
230impl<const N: usize> Simd<f64, N> {
231    #[inline]
232    pub(crate) const fn to_bits(self) -> Simd<u64, N> {
233        assert!(size_of::<Self>() == size_of::<Simd<u64, N>>());
234        unsafe { crate::mem::transmute_copy(&self) }
235    }
236
237    #[inline]
238    pub(crate) const fn from_bits(bits: Simd<u64, N>) -> Self {
239        assert!(size_of::<Self>() == size_of::<Simd<u64, N>>());
240        unsafe { crate::mem::transmute_copy(&bits) }
241    }
242}
243
244#[repr(simd)]
245#[derive(Copy)]
246pub(crate) struct SimdM<T: SimdElement, const N: usize>([T; N]);
247
248impl<T: SimdElement, const N: usize> SimdM<T, N> {
249    #[inline(always)]
250    const fn bool_to_internal(x: bool) -> T {
251        // SAFETY: `T` implements `SimdElement`, so all bit patterns are valid.
252        let ones = const {
253            // Ideally, this would be `transmute([0xFFu8; size_of::<T>()])`, but
254            // `size_of::<T>()` is not allowed to use a generic parameter there.
255            let mut r = crate::mem::MaybeUninit::<T>::uninit();
256            let mut i = 0;
257            while i < crate::mem::size_of::<T>() {
258                r.as_bytes_mut()[i] = crate::mem::MaybeUninit::new(0xFF);
259                i += 1;
260            }
261            unsafe { r.assume_init() }
262        };
263        [T::ZERO, ones][x as usize]
264    }
265
266    #[inline]
267    pub(crate) const fn from_array(elements: [bool; N]) -> Self {
268        let mut internal = [T::ZERO; N];
269        let mut i = 0;
270        while i < N {
271            internal[i] = Self::bool_to_internal(elements[i]);
272            i += 1;
273        }
274        Self(internal)
275    }
276
277    #[inline]
278    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
279    pub(crate) const fn splat(value: bool) -> Self {
280        unsafe { crate::intrinsics::simd::simd_splat(Self::bool_to_internal(value)) }
281    }
282
283    #[inline]
284    pub(crate) const fn as_array(&self) -> &[T; N] {
285        let simd_ptr: *const Self = self;
286        let array_ptr: *const [T; N] = simd_ptr.cast();
287        // SAFETY: We can always read the prefix of a simd type as an array.
288        // There might be more padding afterwards for some widths, but
289        // that's not a problem for reading less than that.
290        unsafe { &*array_ptr }
291    }
292}
293
294// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::SimdM is banned by MCP#838"
295impl<T: SimdElement, const N: usize> Clone for SimdM<T, N> {
296    #[inline]
297    fn clone(&self) -> Self {
298        *self
299    }
300}
301
302#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
303#[rustfmt::skip] // FIXME: https://github.com/rust-lang/stdarch/pull/2133#issuecomment-4524350350
304impl<T: SimdElement, const N: usize> const crate::cmp::PartialEq for SimdM<T, N> {
305    #[inline]
306    fn eq(&self, other: &Self) -> bool {
307        self.as_array() == other.as_array()
308    }
309}
310
311impl<T: SimdElement, const N: usize> crate::fmt::Debug for SimdM<T, N> {
312    #[inline]
313    fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result {
314        debug_simd_finish(f, "SimdM", self.as_array())
315    }
316}
317
318// 16-bit wide types:
319
320pub(crate) type u8x2 = Simd<u8, 2>;
321pub(crate) type i8x2 = Simd<i8, 2>;
322
323// 32-bit wide types:
324
325pub(crate) type u8x4 = Simd<u8, 4>;
326pub(crate) type u16x2 = Simd<u16, 2>;
327
328pub(crate) type i8x4 = Simd<i8, 4>;
329pub(crate) type i16x2 = Simd<i16, 2>;
330
331// 64-bit wide types:
332
333pub(crate) type u8x8 = Simd<u8, 8>;
334pub(crate) type u16x4 = Simd<u16, 4>;
335pub(crate) type u32x2 = Simd<u32, 2>;
336pub(crate) type u64x1 = Simd<u64, 1>;
337
338pub(crate) type i8x8 = Simd<i8, 8>;
339pub(crate) type i16x4 = Simd<i16, 4>;
340pub(crate) type i32x2 = Simd<i32, 2>;
341pub(crate) type i64x1 = Simd<i64, 1>;
342
343pub(crate) type f16x4 = Simd<f16, 4>;
344pub(crate) type f32x2 = Simd<f32, 2>;
345pub(crate) type f64x1 = Simd<f64, 1>;
346
347// 128-bit wide types:
348
349pub(crate) type u8x16 = Simd<u8, 16>;
350pub(crate) type u16x8 = Simd<u16, 8>;
351pub(crate) type u32x4 = Simd<u32, 4>;
352pub(crate) type u64x2 = Simd<u64, 2>;
353
354pub(crate) type i8x16 = Simd<i8, 16>;
355pub(crate) type i16x8 = Simd<i16, 8>;
356pub(crate) type i32x4 = Simd<i32, 4>;
357pub(crate) type i64x2 = Simd<i64, 2>;
358
359pub(crate) type f16x8 = Simd<f16, 8>;
360pub(crate) type f32x4 = Simd<f32, 4>;
361pub(crate) type f64x2 = Simd<f64, 2>;
362
363pub(crate) type m8x16 = SimdM<i8, 16>;
364pub(crate) type m16x8 = SimdM<i16, 8>;
365pub(crate) type m32x4 = SimdM<i32, 4>;
366pub(crate) type m64x2 = SimdM<i64, 2>;
367
368// 256-bit wide types:
369
370pub(crate) type u8x32 = Simd<u8, 32>;
371pub(crate) type u16x16 = Simd<u16, 16>;
372pub(crate) type u32x8 = Simd<u32, 8>;
373pub(crate) type u64x4 = Simd<u64, 4>;
374
375pub(crate) type i8x32 = Simd<i8, 32>;
376pub(crate) type i16x16 = Simd<i16, 16>;
377pub(crate) type i32x8 = Simd<i32, 8>;
378pub(crate) type i64x4 = Simd<i64, 4>;
379
380pub(crate) type f16x16 = Simd<f16, 16>;
381pub(crate) type f32x8 = Simd<f32, 8>;
382pub(crate) type f64x4 = Simd<f64, 4>;
383
384pub(crate) type m8x32 = SimdM<i8, 32>;
385pub(crate) type m16x16 = SimdM<i16, 16>;
386pub(crate) type m32x8 = SimdM<i32, 8>;
387
388// 512-bit wide types:
389
390pub(crate) type u8x64 = Simd<u8, 64>;
391pub(crate) type u16x32 = Simd<u16, 32>;
392pub(crate) type u32x16 = Simd<u32, 16>;
393pub(crate) type u64x8 = Simd<u64, 8>;
394
395pub(crate) type i8x64 = Simd<i8, 64>;
396pub(crate) type i16x32 = Simd<i16, 32>;
397pub(crate) type i32x16 = Simd<i32, 16>;
398pub(crate) type i64x8 = Simd<i64, 8>;
399
400pub(crate) type f16x32 = Simd<f16, 32>;
401pub(crate) type f32x16 = Simd<f32, 16>;
402pub(crate) type f64x8 = Simd<f64, 8>;
403
404// 1024-bit wide types:
405
406pub(crate) type u16x64 = Simd<u16, 64>;
407pub(crate) type u32x32 = Simd<u32, 32>;
408
409pub(crate) type i32x32 = Simd<i32, 32>;
410
411/// Used to continue `Debug`ging SIMD types as `MySimd(1, 2, 3, 4)`, as they
412/// were before moving to array-based simd.
413#[inline]
414pub(crate) fn debug_simd_finish<T: crate::fmt::Debug, const N: usize>(
415    formatter: &mut crate::fmt::Formatter<'_>,
416    type_name: &str,
417    array: &[T; N],
418) -> crate::fmt::Result {
419    crate::fmt::Formatter::debug_tuple_fields_finish(
420        formatter,
421        type_name,
422        &crate::array::from_fn::<&dyn crate::fmt::Debug, N, _>(|i| &array[i]),
423    )
424}
core/stdarch/crates/core_arch/src/simd.rs

core/stdarch/crates/core_arch/src/
simd.rs
