core/stdarch/crates/core_arch/src/x86_64/

amx.rs

use crate::core_arch::x86_64::{__tile1024i, Tile};
use crate::core_arch::{simd::*, x86::*};

#[cfg(test)]
use stdarch_test::assert_instr;

/// Load tile configuration from a 64-byte memory location specified by `mem_addr`.
/// The tile configuration format is specified below, and includes the tile type pallette,
/// the number of bytes per row, and the number of rows. If the specified pallette_id is zero,
/// that signifies the init state for both the tile config and the tile data, and the tiles are zeroed.
/// Any invalid configurations will result in #GP fault.
///
/// ```intel
/// //	format of memory payload. each field is a byte.
///		 0: palette
///		 1: start_row
///	  2-15: reserved, must be zero
///	 16-17: tile0.colsb
///	 18-19: tile1.colsb
///	 20-21: tile2.colsb
///			...
///	 30-31: tile7.colsb
///	 32-47: reserved, must be zero
///		48: tile0.rows
///		49: tile1.rows
///		50: tile2.rows
///			 ...
///		55: tile7.rows
///	 56-63: reserved, must be zero
/// ```
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_loadconfig&ig_expand=6875)
#[inline]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(ldtilecfg))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_loadconfig(mem_addr: *const u8) {
    ldtilecfg(mem_addr);
}

/// Stores the current tile configuration to a 64-byte memory location specified by `mem_addr`.
/// The tile configuration format is as specified in [`_tile_loadconfig`], and includes the tile type pallette,
/// the number of bytes per row, and the number of rows. If tiles are not configured, all zeroes will be stored to memory.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_storeconfig&ig_expand=6879)
#[inline]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(sttilecfg))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_storeconfig(mem_addr: *mut u8) {
    sttilecfg(mem_addr);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst using the tile configuration previously configured via [`_tile_loadconfig`].
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_loadd&ig_expand=6877)
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tileloadd, DST = 0))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_loadd<const DST: i32>(base: *const u8, stride: usize) {
    static_assert_uimm_bits!(DST, 3);
    tileloadd64(DST as i8, base, stride as u64);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst. The shape
/// of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_loadd&ig_expand=6877)
#[inline]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tileloadd))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_loadd(dst: *mut __tile1024i, base: *const u8, stride: usize) {
    (*dst).tile = tileloadd64_internal((*dst).rows, (*dst).colsb, base, stride as u64);
}

/// Release the tile configuration to return to the init state, which releases all storage it currently holds.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_release&ig_expand=6878)
#[inline]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tilerelease))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_release() {
    tilerelease();
}

/// Store the tile specified by src to memory specified by base address and stride using the tile configuration previously configured via [`_tile_loadconfig`].
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_stored&ig_expand=6881)
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tilestored, DST = 0))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_stored<const DST: i32>(base: *mut u8, stride: usize) {
    static_assert_uimm_bits!(DST, 3);
    tilestored64(DST as i8, base, stride as u64);
}

/// Store the tile specified by src to memory specified by base address and stride. The shape of the tile
/// is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_stored&ig_expand=6881)
#[inline]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tilestored))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_stored(base: *mut u8, stride: usize, src: __tile1024i) {
    tilestored64_internal(src.rows, src.colsb, base, stride as u64, src.tile);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst using the tile configuration
/// previously configured via [`_tile_loadconfig`]. This intrinsic provides a hint to the implementation that the data will
/// likely not be reused in the near future and the data caching can be optimized accordingly.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_stream_loadd&ig_expand=6883)
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tileloaddt1, DST = 0))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_stream_loadd<const DST: i32>(base: *const u8, stride: usize) {
    static_assert_uimm_bits!(DST, 3);
    tileloaddt164(DST as i8, base, stride as u64);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst. The shape
/// of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
/// This intrinsic provides a hint to the implementation that the data will likely not be reused in the
/// near future and the data caching can be optimized accordingly.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_stream_loadd&ig_expand=6883)
#[inline]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tileloaddt1))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_stream_loadd(dst: *mut __tile1024i, base: *const u8, stride: usize) {
    (*dst).tile = tileloaddt164_internal((*dst).rows, (*dst).colsb, base, stride as u64);
}

/// Zero the tile specified by `tdest`.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_zero&ig_expand=6885)
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tilezero, DST = 0))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_zero<const DST: i32>() {
    static_assert_uimm_bits!(DST, 3);
    tilezero(DST as i8);
}

/// Zero the tile specified by `dst`. The shape of the tile is specified in the struct of [`__tile1024i`].
/// The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_zero&ig_expand=6885)
#[inline]
#[target_feature(enable = "amx-tile")]
#[cfg_attr(test, assert_instr(tilezero))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_zero(dst: *mut __tile1024i) {
    (*dst).tile = tilezero_internal((*dst).rows, (*dst).colsb);
}

/// Compute dot-product of BF16 (16-bit) floating-point pairs in tiles a and b,
/// accumulating the intermediate single-precision (32-bit) floating-point elements
/// with elements in dst, and store the 32-bit result back to tile dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbf16ps&ig_expand=6864)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-bf16")]
#[cfg_attr(test, assert_instr(tdpbf16ps, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpbf16ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpbf16ps(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles a and b,
/// accumulating the intermediate single-precision (32-bit) floating-point elements
/// with elements in dst, and store the 32-bit result back to tile dst. The shape of the tile
/// is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_dpbf16ps&ig_expand=6864)
#[inline]
#[target_feature(enable = "amx-bf16")]
#[cfg_attr(test, assert_instr(tdpbf16ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpbf16ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpbf16ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding
/// signed 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbssd&ig_expand=6866)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbssd, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpbssd<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpbssd(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding
/// signed 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_dpbssd&ig_expand=6866)
#[inline]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbssd))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpbssd(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpbssd_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding
/// unsigned 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbsud&ig_expand=6868)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbsud, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpbsud<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpbsud(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of signed 8-bit integers in a with corresponding
/// unsigned 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_dpbsud&ig_expand=6868)
#[inline]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbsud))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpbsud(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpbsud_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding
/// signed 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbusd&ig_expand=6870)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbusd, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpbusd<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpbusd(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding
/// signed 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_dpbusd&ig_expand=6870)
#[inline]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbusd))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpbusd(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpbusd_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding
/// unsigned 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpbuud&ig_expand=6872)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbuud, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpbuud<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpbuud(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of bytes in tiles with a source/destination accumulator.
/// Multiply groups of 4 adjacent pairs of unsigned 8-bit integers in a with corresponding
/// unsigned 8-bit integers in b, producing 4 intermediate 32-bit results.
/// Sum these 4 results with the corresponding 32-bit integer in dst, and store the 32-bit result back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_dpbuud&ig_expand=6872)
#[inline]
#[target_feature(enable = "amx-int8")]
#[cfg_attr(test, assert_instr(tdpbuud))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpbuud(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpbuud_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles a and b,
/// accumulating the intermediate single-precision (32-bit) floating-point elements
///  with elements in dst, and store the 32-bit result back to tile dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_dpfp16ps&ig_expand=6874)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-fp16")]
#[cfg_attr(test, assert_instr(tdpfp16ps, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpfp16ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpfp16ps(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of FP16 (16-bit) floating-point pairs in tiles a and b,
/// accumulating the intermediate single-precision (32-bit) floating-point elements
///  with elements in dst, and store the 32-bit result back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_dpfp16ps&ig_expand=6874)
#[inline]
#[target_feature(enable = "amx-fp16")]
#[cfg_attr(test, assert_instr(tdpfp16ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpfp16ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpfp16ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Perform matrix multiplication of two tiles containing complex elements and accumulate the results into a packed single precision tile.
/// Each dword element in input tiles a and b is interpreted as a complex number with FP16 real part and FP16 imaginary part.
/// Calculates the imaginary part of the result. For each possible combination of (row of a, column of b),
/// it performs a set of multiplication and accumulations on all corresponding complex numbers (one from a and one from b).
/// The imaginary part of the a element is multiplied with the real part of the corresponding b element, and the real part of
/// the a element is multiplied with the imaginary part of the corresponding b elements. The two accumulated results are added,
/// and then accumulated into the corresponding row and column of dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_cmmimfp16ps&ig_expand=6860)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-complex")]
#[cfg_attr(test, assert_instr(tcmmimfp16ps, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cmmimfp16ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tcmmimfp16ps(DST as i8, A as i8, B as i8);
}

/// Perform matrix multiplication of two tiles containing complex elements and accumulate the results into a packed single precision tile.
/// Each dword element in input tiles a and b is interpreted as a complex number with FP16 real part and FP16 imaginary part.
/// Calculates the imaginary part of the result. For each possible combination of (row of a, column of b),
/// it performs a set of multiplication and accumulations on all corresponding complex numbers (one from a and one from b).
/// The imaginary part of the a element is multiplied with the real part of the corresponding b element, and the real part of
/// the a element is multiplied with the imaginary part of the corresponding b elements. The two accumulated results are added,
/// and then accumulated into the corresponding row and column of dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_cmmimfp16ps&ig_expand=6860)
#[inline]
#[target_feature(enable = "amx-complex")]
#[cfg_attr(test, assert_instr(tcmmimfp16ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_cmmimfp16ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tcmmimfp16ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Perform matrix multiplication of two tiles containing complex elements and accumulate the results into a packed single precision tile.
/// Each dword element in input tiles a and b is interpreted as a complex number with FP16 real part and FP16 imaginary part.
/// Calculates the real part of the result. For each possible combination of (row of a, column of b),
/// it performs a set of multiplication and accumulations on all corresponding complex numbers (one from a and one from b).
/// The real part of the a element is multiplied with the real part of the corresponding b element, and the negated imaginary part of
/// the a element is multiplied with the imaginary part of the corresponding b elements.
/// The two accumulated results are added, and then accumulated into the corresponding row and column of dst.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_tile_cmmrlfp16ps&ig_expand=6862)
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-complex")]
#[cfg_attr(test, assert_instr(tcmmrlfp16ps, DST = 0, A = 1, B = 2))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cmmrlfp16ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tcmmrlfp16ps(DST as i8, A as i8, B as i8);
}

/// Perform matrix multiplication of two tiles containing complex elements and accumulate the results into a packed single precision tile.
/// Each dword element in input tiles a and b is interpreted as a complex number with FP16 real part and FP16 imaginary part.
/// Calculates the real part of the result. For each possible combination of (row of a, column of b),
/// it performs a set of multiplication and accumulations on all corresponding complex numbers (one from a and one from b).
/// The real part of the a element is multiplied with the real part of the corresponding b element, and the negated imaginary part of
/// the a element is multiplied with the imaginary part of the corresponding b elements.
/// The two accumulated results are added, and then accumulated into the corresponding row and column of dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
///
/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=__tile_cmmrlfp16ps&ig_expand=6862)
#[inline]
#[target_feature(enable = "amx-complex")]
#[cfg_attr(test, assert_instr(tcmmrlfp16ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_cmmrlfp16ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tcmmrlfp16ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of BF8 (8-bit E5M2) floating-point elements in tile a and BF8 (8-bit E5M2)
/// floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tdpbf8ps, DST = 0, A = 1, B = 2)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpbf8ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpbf8ps(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of BF8 (8-bit E5M2) floating-point elements in tile a and BF8 (8-bit E5M2)
/// floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tdpbf8ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpbf8ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpbf8ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of BF8 (8-bit E5M2) floating-point elements in tile a and HF8
/// (8-bit E4M3) floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tdpbhf8ps, DST = 0, A = 1, B = 2)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dpbhf8ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdpbhf8ps(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of BF8 (8-bit E5M2) floating-point elements in tile a and HF8
/// (8-bit E4M3) floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tdpbhf8ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dpbhf8ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdpbhf8ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of HF8 (8-bit E4M3) floating-point elements in tile a and BF8
/// (8-bit E5M2) floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tdphbf8ps, DST = 0, A = 1, B = 2)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dphbf8ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdphbf8ps(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of HF8 (8-bit E4M3) floating-point elements in tile a and BF8
/// (8-bit E5M2) floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tdphbf8ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dphbf8ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdphbf8ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Compute dot-product of HF8 (8-bit E4M3) floating-point elements in tile a and HF8 (8-bit E4M3)
/// floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
#[inline]
#[rustc_legacy_const_generics(0, 1, 2)]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tdphf8ps, DST = 0, A = 1, B = 2)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_dphf8ps<const DST: i32, const A: i32, const B: i32>() {
    static_assert_uimm_bits!(DST, 3);
    static_assert_uimm_bits!(A, 3);
    static_assert_uimm_bits!(B, 3);
    tdphf8ps(DST as i8, A as i8, B as i8);
}

/// Compute dot-product of HF8 (8-bit E4M3) floating-point elements in tile a and HF8 (8-bit E4M3)
/// floating-point elements in tile b, accumulating the intermediate single-precision
/// (32-bit) floating-point elements with elements in dst, and store the 32-bit result
/// back to tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-fp8")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tdphf8ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_dphf8ps(dst: *mut __tile1024i, a: __tile1024i, b: __tile1024i) {
    (*dst).tile = tdphf8ps_internal(a.rows, b.colsb, a.colsb, (*dst).tile, a.tile, b.tile);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst
/// using the tile configuration previously configured via [`_tile_loadconfig`].
/// Additionally, this intrinsic indicates the source memory location is likely to become
/// read-shared by multiple processors, i.e., read in the future by at least one other processor
/// before it is written, assuming it is ever written in the future.
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-movrs")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tileloaddrs, DST = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_loaddrs<const DST: i32>(base: *const u8, stride: usize) {
    static_assert_uimm_bits!(DST, 3);
    tileloaddrs64(DST as i8, base, stride as u64);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
/// Additionally, this intrinsic indicates the source memory location is likely to become
/// read-shared by multiple processors, i.e., read in the future by at least one other processor
/// before it is written, assuming it is ever written in the future.
#[inline]
#[target_feature(enable = "amx-movrs")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tileloaddrs))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_loaddrs(dst: *mut __tile1024i, base: *const u8, stride: usize) {
    (*dst).tile = tileloaddrs64_internal((*dst).rows, (*dst).colsb, base, stride as u64);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst
/// using the tile configuration previously configured via [`_tile_loadconfig`].
/// Provides a hint to the implementation that the data would be reused but does not need
/// to be resident in the nearest cache levels.
/// Additionally, this intrinsic indicates the source memory location is likely to become
/// read-shared by multiple processors, i.e., read in the future by at least one other processor
/// before it is written, assuming it is ever written in the future.
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-movrs")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tileloaddrst1, DST = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_stream_loaddrs<const DST: i32>(base: *const u8, stride: usize) {
    static_assert_uimm_bits!(DST, 3);
    tileloaddrst164(DST as i8, base, stride as u64);
}

/// Load tile rows from memory specified by base address and stride into destination tile dst.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
/// Provides a hint to the implementation that the data would be reused but does not need
/// to be resident in the nearest cache levels.
/// Additionally, this intrinsic indicates the source memory location is likely to become
/// read-shared by multiple processors, i.e., read in the future by at least one other processor
/// before it is written, assuming it is ever written in the future.
#[inline]
#[target_feature(enable = "amx-movrs")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tileloaddrst1))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_stream_loaddrs(dst: *mut __tile1024i, base: *const u8, stride: usize) {
    (*dst).tile = tileloaddrst164_internal((*dst).rows, (*dst).colsb, base, stride as u64);
}

/// Moves a row from a tile register to a zmm register, converting the packed 32-bit signed integer
/// elements to packed single-precision (32-bit) floating-point elements.
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowd2ps, TILE = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowd2ps<const TILE: i32>(row: u32) -> __m512 {
    static_assert_uimm_bits!(TILE, 3);
    tcvtrowd2ps(TILE as i8, row).as_m512()
}

/// Moves a row from a tile register to a zmm register, converting the packed 32-bit signed integer
/// elements to packed single-precision (32-bit) floating-point elements.
#[inline]
#[rustc_legacy_const_generics(0, 1)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowd2ps, TILE = 0, ROW = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowd2psi<const TILE: i32, const ROW: i32>() -> __m512 {
    static_assert_uimm_bits!(TILE, 3);
    static_assert_uimm_bits!(ROW, 6);
    tcvtrowd2psi(TILE as i8, ROW as u32).as_m512()
}

/// Moves a row from a tile register to a zmm register, converting the packed 32-bit signed integer
/// elements to packed single-precision (32-bit) floating-point elements.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tcvtrowd2ps))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_cvtrowd2ps(src: __tile1024i, row: u32) -> __m512 {
    tcvtrowd2ps_internal(src.rows, src.colsb, src.tile, row).as_m512()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the high 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2phh, TILE = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2phh<const TILE: i32>(row: u32) -> __m512h {
    static_assert_uimm_bits!(TILE, 3);
    tcvtrowps2phh(TILE as i8, row).as_m512h()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the high 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0, 1)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2phh, TILE = 0, ROW = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2phhi<const TILE: i32, const ROW: i32>() -> __m512h {
    static_assert_uimm_bits!(TILE, 3);
    static_assert_uimm_bits!(ROW, 6);
    tcvtrowps2phhi(TILE as i8, ROW as u32).as_m512h()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the high 16-bits within each 32-bit element of the returned vector.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tcvtrowps2phh))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_cvtrowps2phh(src: __tile1024i, row: u32) -> __m512h {
    tcvtrowps2phh_internal(src.rows, src.colsb, src.tile, row).as_m512h()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the low 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2phl, TILE = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2phl<const TILE: i32>(row: u32) -> __m512h {
    static_assert_uimm_bits!(TILE, 3);
    tcvtrowps2phl(TILE as i8, row).as_m512h()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the low 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0, 1)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2phl, TILE = 0, ROW = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2phli<const TILE: i32, const ROW: i32>() -> __m512h {
    static_assert_uimm_bits!(TILE, 3);
    static_assert_uimm_bits!(ROW, 6);
    tcvtrowps2phli(TILE as i8, ROW as u32).as_m512h()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed half-precision (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the low 16-bits within each 32-bit element of the returned vector.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tcvtrowps2phl))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_cvtrowps2phl(src: __tile1024i, row: u32) -> __m512h {
    tcvtrowps2phl_internal(src.rows, src.colsb, src.tile, row).as_m512h()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed BF16 (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the high 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2bf16h, TILE = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2bf16h<const TILE: i32>(row: u32) -> __m512bh {
    static_assert_uimm_bits!(TILE, 3);
    tcvtrowps2bf16h(TILE as i8, row).as_m512bh()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed BF16 (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the high 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0, 1)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2bf16h, TILE = 0, ROW = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2bf16hi<const TILE: i32, const ROW: i32>() -> __m512bh {
    static_assert_uimm_bits!(TILE, 3);
    static_assert_uimm_bits!(ROW, 6);
    tcvtrowps2bf16hi(TILE as i8, ROW as u32).as_m512bh()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed BF16 (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the high 16-bits within each 32-bit element of the returned vector.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tcvtrowps2bf16h))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_cvtrowps2bf16h(src: __tile1024i, row: u32) -> __m512bh {
    tcvtrowps2bf16h_internal(src.rows, src.colsb, src.tile, row).as_m512bh()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed BF16 (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the low 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2bf16l, TILE = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2bf16l<const TILE: i32>(row: u32) -> __m512bh {
    static_assert_uimm_bits!(TILE, 3);
    tcvtrowps2bf16l(TILE as i8, row).as_m512bh()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed BF16 (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the low 16-bits within each 32-bit element of the returned vector.
#[inline]
#[rustc_legacy_const_generics(0, 1)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tcvtrowps2bf16l, TILE = 0, ROW = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_cvtrowps2bf16li<const TILE: i32, const ROW: i32>() -> __m512bh {
    static_assert_uimm_bits!(TILE, 3);
    static_assert_uimm_bits!(ROW, 6);
    tcvtrowps2bf16li(TILE as i8, ROW as u32).as_m512bh()
}

/// Moves a row from a tile register to a zmm register, converting the packed single-precision (32-bit)
/// floating-point elements to packed BF16 (16-bit) floating-point elements. The resulting
/// 16-bit elements are placed in the low 16-bits within each 32-bit element of the returned vector.
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tcvtrowps2bf16l))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_cvtrowps2bf16l(src: __tile1024i, row: u32) -> __m512bh {
    tcvtrowps2bf16l_internal(src.rows, src.colsb, src.tile, row).as_m512bh()
}

/// Moves one row of tile data into a zmm vector register
#[inline]
#[rustc_legacy_const_generics(0)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tilemovrow, TILE = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_movrow<const TILE: i32>(row: u32) -> __m512i {
    static_assert_uimm_bits!(TILE, 3);
    tilemovrow(TILE as i8, row).as_m512i()
}

/// Moves one row of tile data into a zmm vector register
#[inline]
#[rustc_legacy_const_generics(0, 1)]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(
    all(test, not(target_vendor = "apple")),
    assert_instr(tilemovrow, TILE = 0, ROW = 0)
)]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn _tile_movrowi<const TILE: i32, const ROW: i32>() -> __m512i {
    static_assert_uimm_bits!(TILE, 3);
    static_assert_uimm_bits!(ROW, 6);
    tilemovrowi(TILE as i8, ROW as u32).as_m512i()
}

/// Moves one row of tile data into a zmm vector register
/// The shape of the tile is specified in the struct of [`__tile1024i`]. The register of the tile is allocated by the compiler.
#[inline]
#[target_feature(enable = "amx-avx512,avx10.2")]
#[cfg_attr(all(test, not(target_vendor = "apple")), assert_instr(tilemovrow))]
#[unstable(feature = "x86_amx_intrinsics", issue = "126622")]
pub unsafe fn __tile_movrow(src: __tile1024i, row: u32) -> __m512i {
    tilemovrow_internal(src.rows, src.colsb, src.tile, row).as_m512i()
}

#[allow(improper_ctypes)]
unsafe extern "unadjusted" {
    #[link_name = "llvm.x86.ldtilecfg"]
    fn ldtilecfg(mem_addr: *const u8);
    #[link_name = "llvm.x86.sttilecfg"]
    fn sttilecfg(mem_addr: *mut u8);

    #[link_name = "llvm.x86.tileloadd64"]
    fn tileloadd64(dst: i8, base: *const u8, stride: u64);
    #[link_name = "llvm.x86.tileloadd64.internal"]
    fn tileloadd64_internal(rows: u16, colsb: u16, base: *const u8, stride: u64) -> Tile;

    #[link_name = "llvm.x86.tileloaddt164"]
    fn tileloaddt164(dst: i8, base: *const u8, stride: u64);
    #[link_name = "llvm.x86.tileloaddt164.internal"]
    fn tileloaddt164_internal(rows: u16, colsb: u16, base: *const u8, stride: u64) -> Tile;

    #[link_name = "llvm.x86.tilerelease"]
    fn tilerelease();

    #[link_name = "llvm.x86.tilestored64"]
    fn tilestored64(dst: i8, base: *mut u8, stride: u64);
    #[link_name = "llvm.x86.tilestored64.internal"]
    fn tilestored64_internal(rows: u16, colsb: u16, base: *mut u8, stride: u64, src: Tile);

    #[link_name = "llvm.x86.tilezero"]
    fn tilezero(dst: i8);
    #[link_name = "llvm.x86.tilezero.internal"]
    fn tilezero_internal(rows: u16, colsb: u16) -> Tile;

    #[link_name = "llvm.x86.tdpbf16ps"]
    fn tdpbf16ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpbf16ps.internal"]
    fn tdpbf16ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdpbuud"]
    fn tdpbuud(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpbuud.internal"]
    fn tdpbuud_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdpbusd"]
    fn tdpbusd(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpbusd.internal"]
    fn tdpbusd_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdpbsud"]
    fn tdpbsud(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpbsud.internal"]
    fn tdpbsud_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdpbssd"]
    fn tdpbssd(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpbssd.internal"]
    fn tdpbssd_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdpfp16ps"]
    fn tdpfp16ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpfp16ps.internal"]
    fn tdpfp16ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tcmmimfp16ps"]
    fn tcmmimfp16ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tcmmimfp16ps.internal"]
    fn tcmmimfp16ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tcmmrlfp16ps"]
    fn tcmmrlfp16ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tcmmrlfp16ps.internal"]
    fn tcmmrlfp16ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdpbf8ps"]
    fn tdpbf8ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpbf8ps.internal"]
    fn tdpbf8ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdpbhf8ps"]
    fn tdpbhf8ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdpbhf8ps.internal"]
    fn tdpbhf8ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdphbf8ps"]
    fn tdphbf8ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdphbf8ps.internal"]
    fn tdphbf8ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tdphf8ps"]
    fn tdphf8ps(dst: i8, a: i8, b: i8);
    #[link_name = "llvm.x86.tdphf8ps.internal"]
    fn tdphf8ps_internal(m: u16, n: u16, k: u16, dst: Tile, a: Tile, b: Tile) -> Tile;

    #[link_name = "llvm.x86.tileloaddrs64"]
    fn tileloaddrs64(dst: i8, base: *const u8, stride: u64);
    #[link_name = "llvm.x86.tileloaddrs64.internal"]
    fn tileloaddrs64_internal(rows: u16, colsb: u16, base: *const u8, stride: u64) -> Tile;

    #[link_name = "llvm.x86.tileloaddrst164"]
    fn tileloaddrst164(dst: i8, base: *const u8, stride: u64);
    #[link_name = "llvm.x86.tileloaddrst164.internal"]
    fn tileloaddrst164_internal(rows: u16, colsb: u16, base: *const u8, stride: u64) -> Tile;

    #[link_name = "llvm.x86.tcvtrowd2ps"]
    fn tcvtrowd2ps(tile: i8, row: u32) -> f32x16;
    #[link_name = "llvm.x86.tcvtrowd2psi"]
    fn tcvtrowd2psi(tile: i8, row: u32) -> f32x16;
    #[link_name = "llvm.x86.tcvtrowd2ps.internal"]
    fn tcvtrowd2ps_internal(rows: u16, colsb: u16, src: Tile, row: u32) -> f32x16;

    #[link_name = "llvm.x86.tcvtrowps2phh"]
    fn tcvtrowps2phh(tile: i8, row: u32) -> f16x32;
    #[link_name = "llvm.x86.tcvtrowps2phhi"]
    fn tcvtrowps2phhi(tile: i8, row: u32) -> f16x32;
    #[link_name = "llvm.x86.tcvtrowps2phh.internal"]
    fn tcvtrowps2phh_internal(rows: u16, colsb: u16, src: Tile, row: u32) -> f16x32;

    #[link_name = "llvm.x86.tcvtrowps2phl"]
    fn tcvtrowps2phl(tile: i8, row: u32) -> f16x32;
    #[link_name = "llvm.x86.tcvtrowps2phli"]
    fn tcvtrowps2phli(tile: i8, row: u32) -> f16x32;
    #[link_name = "llvm.x86.tcvtrowps2phl.internal"]
    fn tcvtrowps2phl_internal(rows: u16, colsb: u16, src: Tile, row: u32) -> f16x32;

    #[link_name = "llvm.x86.tcvtrowps2bf16h"]
    fn tcvtrowps2bf16h(tile: i8, row: u32) -> u16x32;
    #[link_name = "llvm.x86.tcvtrowps2bf16hi"]
    fn tcvtrowps2bf16hi(tile: i8, row: u32) -> u16x32;
    #[link_name = "llvm.x86.tcvtrowps2bf16h.internal"]
    fn tcvtrowps2bf16h_internal(rows: u16, colsb: u16, src: Tile, row: u32) -> u16x32;

    #[link_name = "llvm.x86.tcvtrowps2bf16l"]
    fn tcvtrowps2bf16l(tile: i8, row: u32) -> u16x32;
    #[link_name = "llvm.x86.tcvtrowps2bf16li"]
    fn tcvtrowps2bf16li(tile: i8, row: u32) -> u16x32;
    #[link_name = "llvm.x86.tcvtrowps2bf16l.internal"]
    fn tcvtrowps2bf16l_internal(rows: u16, colsb: u16, src: Tile, row: u32) -> u16x32;

    #[link_name = "llvm.x86.tilemovrow"]
    fn tilemovrow(tile: i8, row: u32) -> i32x16;
    #[link_name = "llvm.x86.tilemovrowi"]
    fn tilemovrowi(tile: i8, row: u32) -> i32x16;
    #[link_name = "llvm.x86.tilemovrow.internal"]
    fn tilemovrow_internal(rows: u16, colsb: u16, src: Tile, row: u32) -> i32x16;
}

#[cfg(test)]
mod tests {
    use crate::core_arch::x86::_mm_cvtness_sbh;
    use crate::core_arch::x86_64::*;
    use core::array;
    use stdarch_test::simd_test;
    #[cfg(target_os = "linux")]
    use syscalls::{Sysno, syscall};

    #[allow(non_camel_case_types)]
    #[repr(C, packed)]
    #[derive(Copy, Clone, Default, Debug, PartialEq)]
    struct __tilecfg {
        /// 0 `or` 1
        palette: u8,
        start_row: u8,
        /// reserved, must be zero
        reserved_a0: [u8; 14],
        /// number of bytes of one row in each tile
        colsb: [u16; 8],
        /// reserved, must be zero
        reserved_b0: [u16; 8],
        /// number of rows in each tile
        rows: [u8; 8],
        /// reserved, must be zero
        reserved_c0: [u8; 8],
    }

    impl __tilecfg {
        fn new(palette: u8, start_row: u8, colsb: [u16; 8], rows: [u8; 8]) -> Self {
            Self {
                palette,
                start_row,
                reserved_a0: [0u8; 14],
                colsb,
                reserved_b0: [0u16; 8],
                rows,
                reserved_c0: [0u8; 8],
            }
        }

        const fn as_ptr(&self) -> *const u8 {
            self as *const Self as *const u8
        }

        fn as_mut_ptr(&mut self) -> *mut u8 {
            self as *mut Self as *mut u8
        }
    }

    #[cfg(not(target_os = "linux"))]
    #[target_feature(enable = "amx-tile")]
    fn _init_amx() {}

    #[cfg(target_os = "linux")]
    #[target_feature(enable = "amx-tile")]
    #[inline]
    fn _init_amx() {
        let mut ret: usize;
        let mut xfeatures: usize = 0;
        ret = unsafe {
            syscall!(Sysno::arch_prctl, 0x1022, &raw mut xfeatures)
                .expect("arch_prctl ARCH_GET_XCOMP_PERM syscall failed")
        };
        if ret != 0 {
            panic!("Failed to get XFEATURES");
        } else {
            match 0b11 & (xfeatures >> 17) {
                0 => panic!("AMX is not available"),
                1 => {
                    ret = unsafe {
                        syscall!(Sysno::arch_prctl, 0x1023, 18)
                            .expect("arch_prctl ARCH_REQ_XCOMP_PERM syscall failed")
                    };
                    if ret != 0 {
                        panic!("Failed to enable AMX");
                    }
                }
                3 => {}
                _ => unreachable!(),
            }
        }
    }

    impl __tile1024i {
        #[inline]
        #[target_feature(enable = "amx-tile")]
        fn zeroed(rows: u16, colsb: u16) -> Self {
            Self {
                rows,
                colsb,
                tile: unsafe { super::tilezero_internal(rows, colsb) },
            }
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test_tile_loadconfig() {
        unsafe {
            let config = __tilecfg::default();
            _tile_loadconfig(config.as_ptr());
            _tile_release();
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test_tile_storeconfig() {
        unsafe {
            let config = __tilecfg::new(1, 0, [32; 8], [8; 8]);
            _tile_loadconfig(config.as_ptr());
            let mut _config = __tilecfg::default();
            _tile_storeconfig(_config.as_mut_ptr());
            _tile_release();
            assert_eq!(config, _config);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test_tile_zero() {
        unsafe {
            _init_amx();
            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            let mut out = [[1_i8; 64]; 16];
            _tile_stored::<0>(out.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(out, [[0; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test__tile_zero() {
        unsafe {
            _init_amx();

            let tile = __tile1024i::zeroed(16, 64);

            let mut out = [[1_i8; 64]; 16];
            __tile_stored(out.as_mut_ptr().cast(), 64, tile);

            assert_eq!(out, [[0; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test_tile_stored() {
        unsafe {
            _init_amx();
            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            let mut out = [[1_i8; 64]; 16];
            _tile_stored::<0>(out.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(out, [[0; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test__tile_stored() {
        unsafe {
            _init_amx();

            let tile = __tile1024i::zeroed(16, 64);

            let mut out = [[1_i8; 64]; 16];
            __tile_stored(out.as_mut_ptr().cast(), 64, tile);

            assert_eq!(out, [[0; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test_tile_loadd() {
        unsafe {
            _init_amx();
            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            let mat = [1_i8; 1024];
            _tile_loadd::<0>(mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            _tile_stored::<0>(out.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test__tile_loadd() {
        unsafe {
            _init_amx();

            let mut tile = __tile1024i::zeroed(16, 64);

            let mat = [1_i8; 1024];
            __tile_loadd(&mut tile, mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            __tile_stored(out.as_mut_ptr().cast(), 64, tile);

            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test_tile_stream_loadd() {
        unsafe {
            _init_amx();
            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            let mat = [1_i8; 1024];
            _tile_stream_loadd::<0>(mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            _tile_stored::<0>(out.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test__tile_stream_loadd() {
        unsafe {
            _init_amx();

            let mut tile = __tile1024i::zeroed(16, 64);

            let mat = [1_i8; 1024];
            __tile_stream_loadd(&mut tile, mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            __tile_stored(out.as_mut_ptr().cast(), 64, tile);

            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-tile")]
    fn test_tile_release() {
        unsafe {
            _tile_release();
        }
    }

    const BF16_1: u16 = 0x3f80;
    const BF16_2: u16 = 0x4000;

    #[simd_test(enable = "amx-bf16")]
    fn test_tile_dpbf16ps() {
        unsafe {
            _init_amx();
            let ones = [BF16_1; 512];
            let twos = [BF16_2; 512];
            let mut res = [[0f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr().cast(), 64);
            _tile_loadd::<2>(twos.as_ptr().cast(), 64);
            _tile_dpbf16ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[64f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-bf16")]
    fn test__tile_dpbf16ps() {
        unsafe {
            _init_amx();
            let ones = [BF16_1; 512];
            let twos = [BF16_2; 512];
            let mut res = [[0f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr().cast(), 64);
            __tile_loadd(&mut b, twos.as_ptr().cast(), 64);
            __tile_dpbf16ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[64f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test_tile_dpbssd() {
        unsafe {
            _init_amx();
            let ones = [-1_i8; 1024];
            let twos = [-2_i8; 1024];
            let mut res = [[0_i32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr().cast(), 64);
            _tile_loadd::<2>(twos.as_ptr().cast(), 64);
            _tile_dpbssd::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test__tile_dpbssd() {
        unsafe {
            _init_amx();
            let ones = [-1_i8; 1024];
            let twos = [-2_i8; 1024];
            let mut res = [[0_i32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr().cast(), 64);
            __tile_loadd(&mut b, twos.as_ptr().cast(), 64);
            __tile_dpbssd(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test_tile_dpbsud() {
        unsafe {
            _init_amx();
            let ones = [-1_i8; 1024];
            let twos = [2_u8; 1024];
            let mut res = [[0_i32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr().cast(), 64);
            _tile_loadd::<2>(twos.as_ptr(), 64);
            _tile_dpbsud::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[-128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test__tile_dpbsud() {
        unsafe {
            _init_amx();
            let ones = [-1_i8; 1024];
            let twos = [2_u8; 1024];
            let mut res = [[0_i32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr().cast(), 64);
            __tile_loadd(&mut b, twos.as_ptr(), 64);
            __tile_dpbsud(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[-128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test_tile_dpbusd() {
        unsafe {
            _init_amx();
            let ones = [1_u8; 1024];
            let twos = [-2_i8; 1024];
            let mut res = [[0_i32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr(), 64);
            _tile_loadd::<2>(twos.as_ptr().cast(), 64);
            _tile_dpbusd::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[-128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test__tile_dpbusd() {
        unsafe {
            _init_amx();
            let ones = [1_u8; 1024];
            let twos = [-2_i8; 1024];
            let mut res = [[0_i32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr(), 64);
            __tile_loadd(&mut b, twos.as_ptr().cast(), 64);
            __tile_dpbusd(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[-128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test_tile_dpbuud() {
        unsafe {
            _init_amx();
            let ones = [1_u8; 1024];
            let twos = [2_u8; 1024];
            let mut res = [[0_i32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr(), 64);
            _tile_loadd::<2>(twos.as_ptr(), 64);
            _tile_dpbuud::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-int8")]
    fn test__tile_dpbuud() {
        unsafe {
            _init_amx();
            let ones = [1_u8; 1024];
            let twos = [2_u8; 1024];
            let mut res = [[0_i32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr(), 64);
            __tile_loadd(&mut b, twos.as_ptr(), 64);
            __tile_dpbuud(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[128_i32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp16")]
    fn test_tile_dpfp16ps() {
        unsafe {
            _init_amx();
            let ones = [1f16; 512];
            let twos = [2f16; 512];
            let mut res = [[0f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr().cast(), 64);
            _tile_loadd::<2>(twos.as_ptr().cast(), 64);
            _tile_dpfp16ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[64f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp16")]
    fn test__tile_dpfp16ps() {
        unsafe {
            _init_amx();
            let ones = [1f16; 512];
            let twos = [2f16; 512];
            let mut res = [[0f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr().cast(), 64);
            __tile_loadd(&mut b, twos.as_ptr().cast(), 64);
            __tile_dpfp16ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[64f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-complex")]
    fn test_tile_cmmimfp16ps() {
        unsafe {
            _init_amx();
            let ones = [1f16; 512];
            let twos = [2f16; 512];
            let mut res = [[0f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr().cast(), 64);
            _tile_loadd::<2>(twos.as_ptr().cast(), 64);
            _tile_cmmimfp16ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[64f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-complex")]
    fn test__tile_cmmimfp16ps() {
        unsafe {
            _init_amx();
            let ones = [1f16; 512];
            let twos = [2f16; 512];
            let mut res = [[0f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr().cast(), 64);
            __tile_loadd(&mut b, twos.as_ptr().cast(), 64);
            __tile_cmmimfp16ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[64f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-complex")]
    fn test_tile_cmmrlfp16ps() {
        unsafe {
            _init_amx();
            let ones = [1f16; 512];
            let twos = [2f16; 512];
            let mut res = [[0f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr().cast(), 64);
            _tile_loadd::<2>(twos.as_ptr().cast(), 64);
            _tile_cmmrlfp16ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[0f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-complex")]
    fn test__tile_cmmrlfp16ps() {
        unsafe {
            _init_amx();
            let ones = [1f16; 512];
            let twos = [2f16; 512];
            let mut res = [[0f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr().cast(), 64);
            __tile_loadd(&mut b, twos.as_ptr().cast(), 64);
            __tile_cmmrlfp16ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[0f32; 16]; 16]);
        }
    }

    const BF8_ONE: u8 = 0x3c;
    const BF8_TWO: u8 = 0x40;
    const HF8_ONE: u8 = 0x38;
    const HF8_TWO: u8 = 0x40;

    #[simd_test(enable = "amx-fp8")]
    fn test_tile_dpbf8ps() {
        unsafe {
            _init_amx();
            let ones = [BF8_ONE; 1024];
            let twos = [BF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr(), 64);
            _tile_loadd::<2>(twos.as_ptr(), 64);
            _tile_dpbf8ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp8")]
    fn test__tile_dpbf8ps() {
        unsafe {
            _init_amx();
            let ones = [BF8_ONE; 1024];
            let twos = [BF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr(), 64);
            __tile_loadd(&mut b, twos.as_ptr(), 64);
            __tile_dpbf8ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp8")]
    fn test_tile_dpbhf8ps() {
        unsafe {
            _init_amx();
            let ones = [BF8_ONE; 1024];
            let twos = [HF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr(), 64);
            _tile_loadd::<2>(twos.as_ptr(), 64);
            _tile_dpbhf8ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp8")]
    fn test__tile_dpbhf8ps() {
        unsafe {
            _init_amx();
            let ones = [BF8_ONE; 1024];
            let twos = [HF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr(), 64);
            __tile_loadd(&mut b, twos.as_ptr(), 64);
            __tile_dpbhf8ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp8")]
    fn test_tile_dphbf8ps() {
        unsafe {
            _init_amx();
            let ones = [HF8_ONE; 1024];
            let twos = [BF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr(), 64);
            _tile_loadd::<2>(twos.as_ptr(), 64);
            _tile_dphbf8ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp8")]
    fn test__tile_dphbf8ps() {
        unsafe {
            _init_amx();
            let ones = [HF8_ONE; 1024];
            let twos = [BF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr(), 64);
            __tile_loadd(&mut b, twos.as_ptr(), 64);
            __tile_dphbf8ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp8")]
    fn test_tile_dphf8ps() {
        unsafe {
            _init_amx();
            let ones = [HF8_ONE; 1024];
            let twos = [HF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];
            let mut config = __tilecfg::default();
            config.palette = 1;
            (0..=2).for_each(|i| {
                config.colsb[i] = 64;
                config.rows[i] = 16;
            });
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            _tile_loadd::<1>(ones.as_ptr(), 64);
            _tile_loadd::<2>(twos.as_ptr(), 64);
            _tile_dphf8ps::<0, 1, 2>();
            _tile_stored::<0>(res.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-fp8")]
    fn test__tile_dphf8ps() {
        unsafe {
            _init_amx();
            let ones = [HF8_ONE; 1024];
            let twos = [HF8_TWO; 1024];
            let mut res = [[0.0_f32; 16]; 16];

            let mut a = __tile1024i::zeroed(16, 64);
            let mut b = __tile1024i::zeroed(16, 64);
            let mut c = __tile1024i::zeroed(16, 64);

            __tile_loadd(&mut a, ones.as_ptr(), 64);
            __tile_loadd(&mut b, twos.as_ptr(), 64);
            __tile_dphf8ps(&mut c, a, b);
            __tile_stored(res.as_mut_ptr().cast(), 64, c);

            assert_eq!(res, [[128.0_f32; 16]; 16]);
        }
    }

    #[simd_test(enable = "amx-movrs")]
    fn test_tile_loaddrs() {
        unsafe {
            _init_amx();
            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            let mat = [1_i8; 1024];
            _tile_loaddrs::<0>(mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            _tile_stored::<0>(out.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-movrs")]
    fn test__tile_loaddrs() {
        unsafe {
            _init_amx();

            let mut tile = __tile1024i::zeroed(16, 64);

            let mat = [1_i8; 1024];
            __tile_loaddrs(&mut tile, mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            __tile_stored(out.as_mut_ptr().cast(), 64, tile);

            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-movrs")]
    fn test_tile_stream_loaddrs() {
        unsafe {
            _init_amx();
            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_zero::<0>();
            let mat = [1_i8; 1024];
            _tile_stream_loaddrs::<0>(mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            _tile_stored::<0>(out.as_mut_ptr().cast(), 64);
            _tile_release();
            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-movrs")]
    fn test__tile_stream_loaddrs() {
        unsafe {
            _init_amx();

            let mut tile = __tile1024i::zeroed(16, 64);

            let mat = [1_i8; 1024];
            __tile_stream_loaddrs(&mut tile, mat.as_ptr().cast(), 64);
            let mut out = [[0_i8; 64]; 16];
            __tile_stored(out.as_mut_ptr().cast(), 64, tile);

            assert_eq!(out, [[1; 64]; 16]);
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_movrow() {
        unsafe {
            _init_amx();
            let array: [[u8; 64]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = _tile_movrow::<0>(i);
                assert_eq!(*row.as_u8x64().as_array(), [i as _; _]);
            }
        }
    }

    macro_rules! wrap_imm4 {
        ($name:ident :: <$TILE:literal>, $row:expr) => {
            match $row {
                0 => $name::<$TILE, 0>(),
                1 => $name::<$TILE, 1>(),
                2 => $name::<$TILE, 2>(),
                3 => $name::<$TILE, 3>(),
                4 => $name::<$TILE, 4>(),
                5 => $name::<$TILE, 5>(),
                6 => $name::<$TILE, 6>(),
                7 => $name::<$TILE, 7>(),
                8 => $name::<$TILE, 8>(),
                9 => $name::<$TILE, 9>(),
                10 => $name::<$TILE, 10>(),
                11 => $name::<$TILE, 11>(),
                12 => $name::<$TILE, 12>(),
                13 => $name::<$TILE, 13>(),
                14 => $name::<$TILE, 14>(),
                15 => $name::<$TILE, 15>(),
                _ => panic!("row index out of range"),
            }
        };
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_movrowi() {
        unsafe {
            _init_amx();
            let array: [[u8; 64]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = wrap_imm4!(_tile_movrowi::<0>, i);
                assert_eq!(*row.as_u8x64().as_array(), [i as _; _]);
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test__tile_movrow() {
        unsafe {
            _init_amx();
            let array: [[u8; 64]; 16] = array::from_fn(|i| [i as _; _]);

            let mut tile = __tile1024i::zeroed(16, 64);
            __tile_loadd(&mut tile, array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = __tile_movrow(tile, i);
                assert_eq!(*row.as_u8x64().as_array(), [i as _; _]);
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowd2ps() {
        unsafe {
            _init_amx();
            let array: [[u32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = _tile_cvtrowd2ps::<0>(i);
                assert_eq!(*row.as_f32x16().as_array(), [i as _; _]);
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowd2psi() {
        unsafe {
            _init_amx();
            let array: [[u32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = wrap_imm4!(_tile_cvtrowd2psi::<0>, i);
                assert_eq!(*row.as_f32x16().as_array(), [i as _; _]);
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test__tile_cvtrowd2ps() {
        unsafe {
            _init_amx();
            let array: [[u32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut tile = __tile1024i::zeroed(16, 64);
            __tile_loadd(&mut tile, array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = __tile_cvtrowd2ps(tile, i);
                assert_eq!(*row.as_f32x16().as_array(), [i as _; _]);
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2phh() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = _tile_cvtrowps2phh::<0>(i);
                assert_eq!(
                    *row.as_f16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 { 0.0 } else { i as _ })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2phhi() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = wrap_imm4!(_tile_cvtrowps2phhi::<0>, i);
                assert_eq!(
                    *row.as_f16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 { 0.0 } else { i as _ })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test__tile_cvtrowps2phh() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut tile = __tile1024i::zeroed(16, 64);
            __tile_loadd(&mut tile, array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = __tile_cvtrowps2phh(tile, i);
                assert_eq!(
                    *row.as_f16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 { 0.0 } else { i as _ })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2phl() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = _tile_cvtrowps2phl::<0>(i);
                assert_eq!(
                    *row.as_f16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 { i as _ } else { 0.0 })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2phli() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = wrap_imm4!(_tile_cvtrowps2phli::<0>, i);
                assert_eq!(
                    *row.as_f16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 { i as _ } else { 0.0 })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test__tile_cvtrowps2phl() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut tile = __tile1024i::zeroed(16, 64);
            __tile_loadd(&mut tile, array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = __tile_cvtrowps2phl(tile, i);
                assert_eq!(
                    *row.as_f16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 { i as _ } else { 0.0 })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2bf16h() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = _tile_cvtrowps2bf16h::<0>(i);
                assert_eq!(
                    *row.as_u16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 {
                        0
                    } else {
                        _mm_cvtness_sbh(i as _).to_bits()
                    })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2bf16hi() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = wrap_imm4!(_tile_cvtrowps2bf16hi::<0>, i);
                assert_eq!(
                    *row.as_u16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 {
                        0
                    } else {
                        _mm_cvtness_sbh(i as _).to_bits()
                    })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test__tile_cvtrowps2bf16h() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut tile = __tile1024i::zeroed(16, 64);
            __tile_loadd(&mut tile, array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = __tile_cvtrowps2bf16h(tile, i);
                assert_eq!(
                    *row.as_u16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 {
                        0
                    } else {
                        _mm_cvtness_sbh(i as _).to_bits()
                    })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2bf16l() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = _tile_cvtrowps2bf16l::<0>(i);
                assert_eq!(
                    *row.as_u16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 {
                        _mm_cvtness_sbh(i as _).to_bits()
                    } else {
                        0
                    })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test_tile_cvtrowps2bf16li() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut config = __tilecfg::default();
            config.palette = 1;
            config.colsb[0] = 64;
            config.rows[0] = 16;
            _tile_loadconfig(config.as_ptr());
            _tile_loadd::<0>(array.as_ptr().cast(), 64);
            for i in 0..16 {
                let row = wrap_imm4!(_tile_cvtrowps2bf16li::<0>, i);
                assert_eq!(
                    *row.as_u16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 {
                        _mm_cvtness_sbh(i as _).to_bits()
                    } else {
                        0
                    })
                );
            }
        }
    }

    #[simd_test(enable = "amx-avx512,avx10.2")]
    fn test__tile_cvtrowps2bf16l() {
        unsafe {
            _init_amx();
            let array: [[f32; 16]; 16] = array::from_fn(|i| [i as _; _]);

            let mut tile = __tile1024i::zeroed(16, 64);
            __tile_loadd(&mut tile, array.as_ptr().cast(), 64);

            for i in 0..16 {
                let row = __tile_cvtrowps2bf16l(tile, i);
                assert_eq!(
                    *row.as_u16x32().as_array(),
                    array::from_fn(|j| if j & 1 == 0 {
                        _mm_cvtness_sbh(i as _).to_bits()
                    } else {
                        0
                    })
                );
            }
        }
    }
}