core/stdarch/crates/core_arch/src/
simd.rs

1//! Internal `#[repr(simd)]` types
2
3#![allow(non_camel_case_types)]
4
5#[inline(always)]
6#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
7pub(crate) const unsafe fn simd_imax<T: Copy>(a: T, b: T) -> T {
8    let mask: T = crate::intrinsics::simd::simd_gt(a, b);
9    crate::intrinsics::simd::simd_select(mask, a, b)
10}
11
12#[inline(always)]
13#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
14pub(crate) const unsafe fn simd_imin<T: Copy>(a: T, b: T) -> T {
15    let mask: T = crate::intrinsics::simd::simd_lt(a, b);
16    crate::intrinsics::simd::simd_select(mask, a, b)
17}
18
19/// SAFETY: All bits patterns must be valid
20pub(crate) unsafe trait SimdElement:
21    Copy + const PartialEq + crate::fmt::Debug
22{
23    // SAFETY: all bits patterns of types implementing this trait must be valid
24    const ZERO: Self = unsafe { crate::mem::zeroed() };
25}
26
27unsafe impl SimdElement for u8 {}
28unsafe impl SimdElement for u16 {}
29unsafe impl SimdElement for u32 {}
30unsafe impl SimdElement for u64 {}
31
32unsafe impl SimdElement for i8 {}
33unsafe impl SimdElement for i16 {}
34unsafe impl SimdElement for i32 {}
35unsafe impl SimdElement for i64 {}
36
37unsafe impl SimdElement for f16 {}
38unsafe impl SimdElement for f32 {}
39unsafe impl SimdElement for f64 {}
40
41#[repr(simd)]
42#[derive(Copy)]
43pub(crate) struct Simd<T: SimdElement, const N: usize>([T; N]);
44
45impl<T: SimdElement, const N: usize> Simd<T, N> {
46    /// A value of this type where all elements are zeroed out.
47    pub(crate) const ZERO: Self = Self::splat(T::ZERO);
48
49    #[inline(always)]
50    pub(crate) const fn from_array(elements: [T; N]) -> Self {
51        Self(elements)
52    }
53
54    #[inline]
55    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
56    pub(crate) const fn splat(value: T) -> Self {
57        unsafe { crate::intrinsics::simd::simd_splat(value) }
58    }
59
60    /// Extract the element at position `index`. Note that `index` is not a constant so this
61    /// operation is not efficient on most platforms. Use for testing only.
62    #[inline]
63    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
64    pub(crate) const fn extract_dyn(&self, index: usize) -> T {
65        assert!(index < N);
66        // SAFETY: self is a vector, T its element type.
67        unsafe { crate::intrinsics::simd::simd_extract_dyn(*self, index as u32) }
68    }
69
70    #[inline]
71    pub(crate) const fn as_array(&self) -> &[T; N] {
72        let simd_ptr: *const Self = self;
73        let array_ptr: *const [T; N] = simd_ptr.cast();
74        // SAFETY: We can always read the prefix of a simd type as an array.
75        // There might be more padding afterwards for some widths, but
76        // that's not a problem for reading less than that.
77        unsafe { &*array_ptr }
78    }
79}
80
81// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::Simd is banned by MCP#838"
82impl<T: SimdElement, const N: usize> Clone for Simd<T, N> {
83    #[inline]
84    fn clone(&self) -> Self {
85        *self
86    }
87}
88
89#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
90impl<T: SimdElement, const N: usize> const crate::cmp::PartialEq for Simd<T, N> {
91    #[inline]
92    fn eq(&self, other: &Self) -> bool {
93        self.as_array() == other.as_array()
94    }
95}
96
97impl<T: SimdElement, const N: usize> crate::fmt::Debug for Simd<T, N> {
98    #[inline]
99    fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result {
100        debug_simd_finish(f, "Simd", self.as_array())
101    }
102}
103
104impl<T: SimdElement> Simd<T, 1> {
105    #[inline]
106    pub(crate) const fn new(x0: T) -> Self {
107        Self([x0])
108    }
109}
110
111impl<T: SimdElement> Simd<T, 2> {
112    #[inline]
113    pub(crate) const fn new(x0: T, x1: T) -> Self {
114        Self([x0, x1])
115    }
116}
117
118impl<T: SimdElement> Simd<T, 4> {
119    #[inline]
120    pub(crate) const fn new(x0: T, x1: T, x2: T, x3: T) -> Self {
121        Self([x0, x1, x2, x3])
122    }
123}
124
125impl<T: SimdElement> Simd<T, 8> {
126    #[inline]
127    pub(crate) const fn new(x0: T, x1: T, x2: T, x3: T, x4: T, x5: T, x6: T, x7: T) -> Self {
128        Self([x0, x1, x2, x3, x4, x5, x6, x7])
129    }
130}
131
132impl<T: SimdElement> Simd<T, 16> {
133    #[inline]
134    pub(crate) const fn new(
135        x0: T,
136        x1: T,
137        x2: T,
138        x3: T,
139        x4: T,
140        x5: T,
141        x6: T,
142        x7: T,
143        x8: T,
144        x9: T,
145        x10: T,
146        x11: T,
147        x12: T,
148        x13: T,
149        x14: T,
150        x15: T,
151    ) -> Self {
152        Self([
153            x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15,
154        ])
155    }
156}
157
158impl<T: SimdElement> Simd<T, 32> {
159    #[inline]
160    pub(crate) const fn new(
161        x0: T,
162        x1: T,
163        x2: T,
164        x3: T,
165        x4: T,
166        x5: T,
167        x6: T,
168        x7: T,
169        x8: T,
170        x9: T,
171        x10: T,
172        x11: T,
173        x12: T,
174        x13: T,
175        x14: T,
176        x15: T,
177        x16: T,
178        x17: T,
179        x18: T,
180        x19: T,
181        x20: T,
182        x21: T,
183        x22: T,
184        x23: T,
185        x24: T,
186        x25: T,
187        x26: T,
188        x27: T,
189        x28: T,
190        x29: T,
191        x30: T,
192        x31: T,
193    ) -> Self {
194        Self([
195            x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x18,
196            x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, x30, x31,
197        ])
198    }
199}
200
201impl<const N: usize> Simd<f16, N> {
202    #[inline]
203    pub(crate) const fn to_bits(self) -> Simd<u16, N> {
204        assert!(size_of::<Self>() == size_of::<Simd<u16, N>>());
205        unsafe { crate::mem::transmute_copy(&self) }
206    }
207
208    #[inline]
209    pub(crate) const fn from_bits(bits: Simd<u16, N>) -> Self {
210        assert!(size_of::<Self>() == size_of::<Simd<u16, N>>());
211        unsafe { crate::mem::transmute_copy(&bits) }
212    }
213}
214
215impl<const N: usize> Simd<f32, N> {
216    #[inline]
217    pub(crate) const fn to_bits(self) -> Simd<u32, N> {
218        assert!(size_of::<Self>() == size_of::<Simd<u32, N>>());
219        unsafe { crate::mem::transmute_copy(&self) }
220    }
221
222    #[inline]
223    pub(crate) const fn from_bits(bits: Simd<u32, N>) -> Self {
224        assert!(size_of::<Self>() == size_of::<Simd<u32, N>>());
225        unsafe { crate::mem::transmute_copy(&bits) }
226    }
227}
228
229impl<const N: usize> Simd<f64, N> {
230    #[inline]
231    pub(crate) const fn to_bits(self) -> Simd<u64, N> {
232        assert!(size_of::<Self>() == size_of::<Simd<u64, N>>());
233        unsafe { crate::mem::transmute_copy(&self) }
234    }
235
236    #[inline]
237    pub(crate) const fn from_bits(bits: Simd<u64, N>) -> Self {
238        assert!(size_of::<Self>() == size_of::<Simd<u64, N>>());
239        unsafe { crate::mem::transmute_copy(&bits) }
240    }
241}
242
243#[repr(simd)]
244#[derive(Copy)]
245pub(crate) struct SimdM<T: SimdElement, const N: usize>([T; N]);
246
247impl<T: SimdElement, const N: usize> SimdM<T, N> {
248    #[inline(always)]
249    const fn bool_to_internal(x: bool) -> T {
250        // SAFETY: `T` implements `SimdElement`, so all bit patterns are valid.
251        let ones = const {
252            // Ideally, this would be `transmute([0xFFu8; size_of::<T>()])`, but
253            // `size_of::<T>()` is not allowed to use a generic parameter there.
254            let mut r = crate::mem::MaybeUninit::<T>::uninit();
255            let mut i = 0;
256            while i < crate::mem::size_of::<T>() {
257                r.as_bytes_mut()[i] = crate::mem::MaybeUninit::new(0xFF);
258                i += 1;
259            }
260            unsafe { r.assume_init() }
261        };
262        [T::ZERO, ones][x as usize]
263    }
264
265    #[inline]
266    pub(crate) const fn from_array(elements: [bool; N]) -> Self {
267        let mut internal = [T::ZERO; N];
268        let mut i = 0;
269        while i < N {
270            internal[i] = Self::bool_to_internal(elements[i]);
271            i += 1;
272        }
273        Self(internal)
274    }
275
276    #[inline]
277    #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
278    pub(crate) const fn splat(value: bool) -> Self {
279        unsafe { crate::intrinsics::simd::simd_splat(Self::bool_to_internal(value)) }
280    }
281
282    #[inline]
283    pub(crate) const fn as_array(&self) -> &[T; N] {
284        let simd_ptr: *const Self = self;
285        let array_ptr: *const [T; N] = simd_ptr.cast();
286        // SAFETY: We can always read the prefix of a simd type as an array.
287        // There might be more padding afterwards for some widths, but
288        // that's not a problem for reading less than that.
289        unsafe { &*array_ptr }
290    }
291}
292
293// `#[derive(Clone)]` causes ICE "Projecting into SIMD type core_arch::simd::SimdM is banned by MCP#838"
294impl<T: SimdElement, const N: usize> Clone for SimdM<T, N> {
295    #[inline]
296    fn clone(&self) -> Self {
297        *self
298    }
299}
300
301#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
302impl<T: SimdElement, const N: usize> const crate::cmp::PartialEq for SimdM<T, N> {
303    #[inline]
304    fn eq(&self, other: &Self) -> bool {
305        self.as_array() == other.as_array()
306    }
307}
308
309impl<T: SimdElement, const N: usize> crate::fmt::Debug for SimdM<T, N> {
310    #[inline]
311    fn fmt(&self, f: &mut crate::fmt::Formatter<'_>) -> crate::fmt::Result {
312        debug_simd_finish(f, "SimdM", self.as_array())
313    }
314}
315
316// 16-bit wide types:
317
318pub(crate) type u8x2 = Simd<u8, 2>;
319pub(crate) type i8x2 = Simd<i8, 2>;
320
321// 32-bit wide types:
322
323pub(crate) type u8x4 = Simd<u8, 4>;
324pub(crate) type u16x2 = Simd<u16, 2>;
325
326pub(crate) type i8x4 = Simd<i8, 4>;
327pub(crate) type i16x2 = Simd<i16, 2>;
328
329// 64-bit wide types:
330
331pub(crate) type u8x8 = Simd<u8, 8>;
332pub(crate) type u16x4 = Simd<u16, 4>;
333pub(crate) type u32x2 = Simd<u32, 2>;
334pub(crate) type u64x1 = Simd<u64, 1>;
335
336pub(crate) type i8x8 = Simd<i8, 8>;
337pub(crate) type i16x4 = Simd<i16, 4>;
338pub(crate) type i32x2 = Simd<i32, 2>;
339pub(crate) type i64x1 = Simd<i64, 1>;
340
341pub(crate) type f16x4 = Simd<f16, 4>;
342pub(crate) type f32x2 = Simd<f32, 2>;
343pub(crate) type f64x1 = Simd<f64, 1>;
344
345// 128-bit wide types:
346
347pub(crate) type u8x16 = Simd<u8, 16>;
348pub(crate) type u16x8 = Simd<u16, 8>;
349pub(crate) type u32x4 = Simd<u32, 4>;
350pub(crate) type u64x2 = Simd<u64, 2>;
351
352pub(crate) type i8x16 = Simd<i8, 16>;
353pub(crate) type i16x8 = Simd<i16, 8>;
354pub(crate) type i32x4 = Simd<i32, 4>;
355pub(crate) type i64x2 = Simd<i64, 2>;
356
357pub(crate) type f16x8 = Simd<f16, 8>;
358pub(crate) type f32x4 = Simd<f32, 4>;
359pub(crate) type f64x2 = Simd<f64, 2>;
360
361pub(crate) type m8x16 = SimdM<i8, 16>;
362pub(crate) type m16x8 = SimdM<i16, 8>;
363pub(crate) type m32x4 = SimdM<i32, 4>;
364pub(crate) type m64x2 = SimdM<i64, 2>;
365
366// 256-bit wide types:
367
368pub(crate) type u8x32 = Simd<u8, 32>;
369pub(crate) type u16x16 = Simd<u16, 16>;
370pub(crate) type u32x8 = Simd<u32, 8>;
371pub(crate) type u64x4 = Simd<u64, 4>;
372
373pub(crate) type i8x32 = Simd<i8, 32>;
374pub(crate) type i16x16 = Simd<i16, 16>;
375pub(crate) type i32x8 = Simd<i32, 8>;
376pub(crate) type i64x4 = Simd<i64, 4>;
377
378pub(crate) type f16x16 = Simd<f16, 16>;
379pub(crate) type f32x8 = Simd<f32, 8>;
380pub(crate) type f64x4 = Simd<f64, 4>;
381
382pub(crate) type m8x32 = SimdM<i8, 32>;
383pub(crate) type m16x16 = SimdM<i16, 16>;
384pub(crate) type m32x8 = SimdM<i32, 8>;
385
386// 512-bit wide types:
387
388pub(crate) type u8x64 = Simd<u8, 64>;
389pub(crate) type u16x32 = Simd<u16, 32>;
390pub(crate) type u32x16 = Simd<u32, 16>;
391pub(crate) type u64x8 = Simd<u64, 8>;
392
393pub(crate) type i8x64 = Simd<i8, 64>;
394pub(crate) type i16x32 = Simd<i16, 32>;
395pub(crate) type i32x16 = Simd<i32, 16>;
396pub(crate) type i64x8 = Simd<i64, 8>;
397
398pub(crate) type f16x32 = Simd<f16, 32>;
399pub(crate) type f32x16 = Simd<f32, 16>;
400pub(crate) type f64x8 = Simd<f64, 8>;
401
402// 1024-bit wide types:
403
404pub(crate) type u16x64 = Simd<u16, 64>;
405pub(crate) type u32x32 = Simd<u32, 32>;
406
407pub(crate) type i32x32 = Simd<i32, 32>;
408
409/// Used to continue `Debug`ging SIMD types as `MySimd(1, 2, 3, 4)`, as they
410/// were before moving to array-based simd.
411#[inline]
412pub(crate) fn debug_simd_finish<T: crate::fmt::Debug, const N: usize>(
413    formatter: &mut crate::fmt::Formatter<'_>,
414    type_name: &str,
415    array: &[T; N],
416) -> crate::fmt::Result {
417    crate::fmt::Formatter::debug_tuple_fields_finish(
418        formatter,
419        type_name,
420        &crate::array::from_fn::<&dyn crate::fmt::Debug, N, _>(|i| &array[i]),
421    )
422}
core/stdarch/crates/core_arch/src/simd.rs

core/stdarch/crates/core_arch/src/
simd.rs
