core/stdarch/crates/core_arch/src/loongarch64/
simd.rs

1//! LoongArch64 SIMD helpers
2
3use self as ls;
4use crate::intrinsics::simd as is;
5
6// Internal extension trait for concrete `Simd<T, N>` types.
7//
8// Provides a small set of helper functionality (`Elem` and `splat`)
9// so generic and macro-based code can operate on different SIMD
10// vector types in a uniform way.
11pub(super) const trait SimdExt: Sized {
12    type Elem;
13
14    unsafe fn splat(v: i64) -> Self;
15}
16
17#[rustfmt::skip] // FIXME: https://github.com/rust-lang/stdarch/pull/2133#issuecomment-4524350350
18macro_rules! impl_simd_ext {
19    ($v:ident, $e:ty) => {
20        #[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
21        const impl SimdExt for crate::core_arch::simd::$v {
22            type Elem = $e;
23
24            #[inline(always)]
25            unsafe fn splat(v: i64) -> Self {
26                is::simd_splat(v as Self::Elem)
27            }
28        }
29    };
30}
31
32impl_simd_ext!(i8x16, i8);
33impl_simd_ext!(i8x32, i8);
34impl_simd_ext!(u8x16, u8);
35impl_simd_ext!(u8x32, u8);
36impl_simd_ext!(i16x8, i16);
37impl_simd_ext!(i16x16, i16);
38impl_simd_ext!(i16x32, i16);
39impl_simd_ext!(u16x8, u16);
40impl_simd_ext!(u16x16, u16);
41impl_simd_ext!(u16x32, u16);
42impl_simd_ext!(i32x4, i32);
43impl_simd_ext!(i32x8, i32);
44impl_simd_ext!(i32x16, i32);
45impl_simd_ext!(u32x4, u32);
46impl_simd_ext!(u32x8, u32);
47impl_simd_ext!(u32x16, u32);
48impl_simd_ext!(i64x2, i64);
49impl_simd_ext!(i64x4, i64);
50impl_simd_ext!(i64x8, i64);
51impl_simd_ext!(u64x2, u64);
52impl_simd_ext!(u64x4, u64);
53impl_simd_ext!(u64x8, u64);
54impl_simd_ext!(i128x2, i128);
55impl_simd_ext!(u128x2, u128);
56impl_simd_ext!(i128x4, i128);
57impl_simd_ext!(u128x4, u128);
58
59#[inline(always)]
60#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
61pub(super) const unsafe fn simd_abs<T: Copy + const SimdExt>(a: T) -> T {
62    let m: T = is::simd_lt(a, ls::simd_splat(0));
63    is::simd_select(m, is::simd_neg(a), a)
64}
65
66#[inline(always)]
67#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
68pub(super) const unsafe fn simd_absd<T: Copy>(a: T, b: T) -> T {
69    let m: T = is::simd_gt(a, b);
70    is::simd_select(m, is::simd_sub(a, b), is::simd_sub(b, a))
71}
72
73#[inline(always)]
74#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
75pub(super) const unsafe fn simd_adda<T: Copy + const SimdExt>(a: T, b: T) -> T {
76    is::simd_add(ls::simd_abs(a), ls::simd_abs(b))
77}
78
79#[inline(always)]
80#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
81pub(super) const unsafe fn simd_andn<T: Copy + const SimdExt>(a: T, b: T) -> T {
82    is::simd_and(ls::simd_not(a), b)
83}
84
85#[inline(always)]
86#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
87pub(super) const unsafe fn simd_bitclr<T: Copy + const SimdExt>(a: T, b: T) -> T {
88    ls::simd_andn(ls::simd_shl(ls::simd_splat(1), b), a)
89}
90
91#[inline(always)]
92#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
93pub(super) const unsafe fn simd_bitrev<T: Copy + const SimdExt>(a: T, b: T) -> T {
94    is::simd_xor(ls::simd_shl(ls::simd_splat(1), b), a)
95}
96
97#[inline(always)]
98#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
99pub(super) const unsafe fn simd_bitset<T: Copy + const SimdExt>(a: T, b: T) -> T {
100    is::simd_or(ls::simd_shl(ls::simd_splat(1), b), a)
101}
102
103#[inline(always)]
104#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
105pub(super) const unsafe fn simd_fmsub<T: Copy>(a: T, b: T, c: T) -> T {
106    is::simd_fma(a, b, is::simd_neg(c))
107}
108
109#[inline(always)]
110#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
111pub(super) const unsafe fn simd_fnmadd<T: Copy>(a: T, b: T, c: T) -> T {
112    is::simd_neg(is::simd_fma(a, b, c))
113}
114
115#[inline(always)]
116#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
117pub(super) const unsafe fn simd_fnmsub<T: Copy>(a: T, b: T, c: T) -> T {
118    is::simd_neg(ls::simd_fmsub(a, b, c))
119}
120
121#[inline(always)]
122#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
123pub(super) const unsafe fn simd_frecip_s<T: Copy>(a: T) -> T {
124    is::simd_div(is::simd_splat(1.0f32), a)
125}
126
127#[inline(always)]
128#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
129pub(super) const unsafe fn simd_frecip_d<T: Copy>(a: T) -> T {
130    is::simd_div(is::simd_splat(1.0f64), a)
131}
132
133#[inline(always)]
134pub(super) unsafe fn simd_frsqrt_s<T: Copy>(a: T) -> T {
135    ls::simd_frecip_s(is::simd_fsqrt(a))
136}
137
138#[inline(always)]
139pub(super) unsafe fn simd_frsqrt_d<T: Copy>(a: T) -> T {
140    ls::simd_frecip_d(is::simd_fsqrt(a))
141}
142
143#[inline(always)]
144#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
145pub(super) const unsafe fn simd_ld<const I: i32, T: Copy>(a: *const i8) -> T {
146    let a = a.offset(I as isize) as *const T;
147    core::ptr::read_unaligned(a)
148}
149
150#[inline(always)]
151#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
152pub(super) const unsafe fn simd_ldx<T: Copy>(a: *const i8, b: i64) -> T {
153    let a = a.offset(b as isize) as *const T;
154    core::ptr::read_unaligned(a)
155}
156
157#[inline(always)]
158#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
159pub(super) const unsafe fn simd_madd<T: Copy>(a: T, b: T, c: T) -> T {
160    is::simd_add(a, is::simd_mul(b, c))
161}
162
163#[inline(always)]
164#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
165pub(super) const unsafe fn simd_msub<T: Copy>(a: T, b: T, c: T) -> T {
166    is::simd_sub(a, is::simd_mul(b, c))
167}
168
169#[inline(always)]
170#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
171pub(super) const unsafe fn simd_muh<T: Copy, W: Copy + const SimdExt>(a: T, b: T) -> T {
172    let a: W = is::simd_cast(a);
173    let b: W = is::simd_cast(b);
174    let p = is::simd_mul(a, b);
175    is::simd_cast(is::simd_shr(
176        p,
177        ls::simd_splat((size_of::<W::Elem>() * 8 / 2) as i64),
178    ))
179}
180
181#[inline(always)]
182#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
183pub(super) const unsafe fn simd_nor<T: Copy + const SimdExt>(a: T, b: T) -> T {
184    ls::simd_not(is::simd_or(a, b))
185}
186
187#[inline(always)]
188#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
189pub(super) const unsafe fn simd_not<T: Copy + const SimdExt>(a: T) -> T {
190    is::simd_xor(a, ls::simd_splat(!0))
191}
192
193#[inline(always)]
194#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
195pub(super) const unsafe fn simd_orn<T: Copy + const SimdExt>(a: T, b: T) -> T {
196    is::simd_or(a, ls::simd_not(b))
197}
198
199#[inline(always)]
200#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
201pub(super) const unsafe fn simd_rotr<T: Copy + const SimdExt>(a: T, b: T) -> T {
202    let m = (size_of::<T::Elem>() * 8 - 1) as i64;
203    let r = is::simd_and(b, ls::simd_splat(m));
204    let l = is::simd_and(is::simd_sub(ls::simd_splat(m + 1), r), ls::simd_splat(m));
205    is::simd_or(is::simd_shr(a, r), is::simd_shl(a, l))
206}
207
208#[inline(always)]
209#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
210pub(super) const unsafe fn simd_shl<T: Copy + const SimdExt>(a: T, b: T) -> T {
211    let m = (size_of::<T::Elem>() * 8 - 1) as i64;
212    is::simd_shl(a, is::simd_and(b, ls::simd_splat(m)))
213}
214
215#[inline(always)]
216#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
217pub(super) const unsafe fn simd_shr<T: Copy + const SimdExt>(a: T, b: T) -> T {
218    let m = (size_of::<T::Elem>() * 8 - 1) as i64;
219    is::simd_shr(a, is::simd_and(b, ls::simd_splat(m)))
220}
221
222#[inline(always)]
223#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
224pub(super) const unsafe fn simd_splat<T: Copy + const SimdExt>(a: i64) -> T {
225    T::splat(a)
226}
227
228#[inline(always)]
229#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
230pub(super) const unsafe fn simd_st<const I: i32, T: Copy>(a: T, b: *mut i8) {
231    let b = b.offset(I as isize) as *mut T;
232    core::ptr::write_unaligned(b, a);
233}
234
235#[inline(always)]
236#[rustc_const_unstable(feature = "stdarch_const_helpers", issue = "none")]
237pub(super) const unsafe fn simd_stx<T: Copy>(a: T, b: *mut i8, c: i64) {
238    let b = b.offset(c as isize) as *mut T;
239    core::ptr::write_unaligned(b, a);
240}
241
242macro_rules! impl_vv {
243    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ty) => {
244        #[inline]
245        #[target_feature(enable = $ft)]
246        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
247        pub fn $name(a: $oty) -> $oty {
248            unsafe {
249                let a: $ity = transmute(a);
250                let r: $ity = $op(a);
251                transmute(r)
252            }
253        }
254    };
255}
256
257pub(super) use impl_vv;
258
259macro_rules! impl_gv {
260    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $gty:ty) => {
261        #[inline]
262        #[target_feature(enable = $ft)]
263        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
264        pub fn $name(a: $gty) -> $oty {
265            unsafe {
266                let r: $ity = $op(a.into());
267                transmute(r)
268            }
269        }
270    };
271}
272
273pub(super) use impl_gv;
274
275macro_rules! impl_ggv {
276    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $gty:ty, $xty:ty, unsafe) => {
277        #[inline]
278        #[target_feature(enable = $ft)]
279        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
280        pub unsafe fn $name(a: $gty, b: $xty) -> $oty {
281            let r: $ity = $op(a, b);
282            transmute(r)
283        }
284    };
285}
286
287pub(super) use impl_ggv;
288
289macro_rules! impl_gsv {
290    ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ident, $gty:ty, $ibs:expr, const, unsafe) => {
291        #[inline]
292        #[target_feature(enable = $ft)]
293        #[rustc_legacy_const_generics(1)]
294        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
295        pub unsafe fn $name<const IMM: i32>(a: $gty) -> $oty {
296            static_assert_simm_bits!(IMM, $ibs);
297            let r: $ity = $op::<IMM, _>(a);
298            transmute(r)
299        }
300    };
301}
302
303pub(super) use impl_gsv;
304
305macro_rules! impl_sv {
306    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $ibs:expr) => {
307        #[inline]
308        #[target_feature(enable = $ft)]
309        #[rustc_legacy_const_generics(0)]
310        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
311        pub fn $name<const IMM: i32>() -> $oty {
312            static_assert_simm_bits!(IMM, $ibs);
313            unsafe {
314                let r: $ity = $op(IMM.into());
315                transmute(r)
316            }
317        }
318    };
319}
320
321pub(super) use impl_sv;
322
323macro_rules! impl_vvv {
324    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ty) => {
325        #[inline]
326        #[target_feature(enable = $ft)]
327        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
328        pub fn $name(a: $oty, b: $oty) -> $oty {
329            unsafe {
330                let a: $ity = transmute(a);
331                let b: $ity = transmute(b);
332                let r: $ity = $op(a, b);
333                transmute(r)
334            }
335        }
336    };
337    ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ty, $wty:ty) => {
338        #[inline]
339        #[target_feature(enable = $ft)]
340        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
341        pub fn $name(a: $oty, b: $oty) -> $oty {
342            unsafe {
343                let a: $ity = transmute(a);
344                let b: $ity = transmute(b);
345                let r: $ity = $op::<$ity, $wty>(a, b);
346                transmute(r)
347            }
348        }
349    };
350}
351
352pub(super) use impl_vvv;
353
354macro_rules! impl_vgg {
355    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $gty:ty, $xty:ty, unsafe) => {
356        #[inline]
357        #[target_feature(enable = $ft)]
358        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
359        pub unsafe fn $name(a: $oty, b: $gty, c: $xty) {
360            $op(a, b, c);
361        }
362    };
363}
364
365pub(super) use impl_vgg;
366
367macro_rules! impl_vgs {
368    ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ident, $gty:ty, $ibs:expr, const, unsafe) => {
369        #[inline]
370        #[target_feature(enable = $ft)]
371        #[rustc_legacy_const_generics(2)]
372        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
373        pub unsafe fn $name<const IMM: i32>(a: $oty, b: $gty) {
374            static_assert_simm_bits!(IMM, $ibs);
375            $op::<IMM, _>(a, b);
376        }
377    };
378}
379
380pub(super) use impl_vgs;
381
382macro_rules! impl_vuv {
383    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident) => {
384        #[inline]
385        #[target_feature(enable = $ft)]
386        #[rustc_legacy_const_generics(1)]
387        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
388        pub fn $name<const IMM: u32>(a: $oty) -> $oty {
389            static_assert_uimm_bits!(IMM, (size_of::<<$ity as SimdExt>::Elem>() * 8).ilog2());
390            unsafe {
391                let a: $ity = transmute(a);
392                let b: $ity = ls::simd_splat(IMM.into());
393                let r: $ity = $op(a, b);
394                transmute(r)
395            }
396        }
397    };
398    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $ibs:expr) => {
399        #[inline]
400        #[target_feature(enable = $ft)]
401        #[rustc_legacy_const_generics(1)]
402        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
403        pub fn $name<const IMM: u32>(a: $oty) -> $oty {
404            static_assert_uimm_bits!(IMM, $ibs);
405            unsafe {
406                let a: $ity = transmute(a);
407                let b: $ity = ls::simd_splat(IMM.into());
408                let r: $ity = $op(a, b);
409                transmute(r)
410            }
411        }
412    };
413    ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ident, $ibs:expr, const) => {
414        #[inline]
415        #[target_feature(enable = $ft)]
416        #[rustc_legacy_const_generics(1)]
417        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
418        pub fn $name<const IMM: u32>(a: $oty) -> $oty {
419            static_assert_uimm_bits!(IMM, $ibs);
420            unsafe {
421                let a: $ity = transmute(a);
422                let r: $ity = $op::<IMM, _>(a);
423                transmute(r)
424            }
425        }
426    };
427}
428
429pub(super) use impl_vuv;
430
431macro_rules! impl_vug {
432    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $gty:ty, $ibs:expr) => {
433        #[inline]
434        #[target_feature(enable = $ft)]
435        #[rustc_legacy_const_generics(1)]
436        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
437        pub fn $name<const IMM: u32>(a: $oty) -> $gty {
438            static_assert_uimm_bits!(IMM, $ibs);
439            unsafe {
440                let a: $ity = transmute(a);
441                let r: <$ity as SimdExt>::Elem = $op(a, IMM);
442                r as $gty
443            }
444        }
445    };
446}
447
448pub(super) use impl_vug;
449
450macro_rules! impl_vsv {
451    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $ibs:expr) => {
452        #[inline]
453        #[target_feature(enable = $ft)]
454        #[rustc_legacy_const_generics(1)]
455        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
456        pub fn $name<const IMM: i32>(a: $oty) -> $oty {
457            static_assert_simm_bits!(IMM, $ibs);
458            unsafe {
459                let a: $ity = transmute(a);
460                let b: $ity = ls::simd_splat(IMM.into());
461                let r: $ity = $op(a, b);
462                transmute(r)
463            }
464        }
465    };
466}
467
468pub(super) use impl_vsv;
469
470macro_rules! impl_vvvv {
471    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ty) => {
472        #[inline]
473        #[target_feature(enable = $ft)]
474        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
475        pub fn $name(a: $oty, b: $oty, c: $oty) -> $oty {
476            unsafe {
477                let a: $ity = transmute(a);
478                let b: $ity = transmute(b);
479                let c: $ity = transmute(c);
480                let r: $ity = $op(a, b, c);
481                transmute(r)
482            }
483        }
484    };
485}
486
487pub(super) use impl_vvvv;
488
489macro_rules! impl_vvuv {
490    ($ft:literal, $name:ident, $op:ident, $oty:ty, $ity:ident, $ibs:expr, const) => {
491        #[inline]
492        #[target_feature(enable = $ft)]
493        #[rustc_legacy_const_generics(2)]
494        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
495        pub fn $name<const IMM: u32>(a: $oty, b: $oty) -> $oty {
496            static_assert_uimm_bits!(IMM, $ibs);
497            unsafe {
498                let a: $ity = transmute(a);
499                let b: $ity = transmute(b);
500                let r: $ity = $op::<IMM, _>(a, b);
501                transmute(r)
502            }
503        }
504    };
505}
506
507pub(super) use impl_vvuv;
508
509macro_rules! impl_vugv {
510    ($ft:literal, $name:ident, $op:path, $oty:ty, $ity:ident, $gty:ty, $ibs:expr) => {
511        #[inline]
512        #[target_feature(enable = $ft)]
513        #[rustc_legacy_const_generics(2)]
514        #[unstable(feature = "stdarch_loongarch", issue = "117427")]
515        pub fn $name<const IMM: u32>(a: $oty, b: $gty) -> $oty {
516            static_assert_uimm_bits!(IMM, $ibs);
517            unsafe {
518                let a: $ity = transmute(a);
519                let r: $ity = $op(a, IMM, b as <$ity as SimdExt>::Elem);
520                transmute(r)
521            }
522        }
523    };
524}
525
526pub(super) use impl_vugv;
core/stdarch/crates/core_arch/src/loongarch64/simd.rs

core/stdarch/crates/core_arch/src/loongarch64/
simd.rs
