From e5961eeb508d13b66654e3a4aa8af4f1fe4e0125 Mon Sep 17 00:00:00 2001 From: dsecurity49 Date: Sat, 5 Sep 2026 10:01:22 +0530 Subject: [PATCH] cranelift: fix missing ISLE lowering rules from #12197 on AArch64, x86_64, and RISC-V --- cranelift/codegen/src/isa/aarch64/lower.isle | 13 +- cranelift/codegen/src/isa/riscv64/lower.isle | 6 + cranelift/codegen/src/isa/x64/inst.isle | 5 + .../isa/aarch64/vhigh-bits-float.clif | 121 ++++++++++++++++++ .../isa/riscv64/select-spectre-vec.clif | 94 ++++++++++++++ .../filetests/isa/x64/vany-true-i8.clif | 36 ++++++ .../runtests/simd-vhighbits-float.clif | 1 + 7 files changed, 272 insertions(+), 4 deletions(-) create mode 100644 cranelift/filetests/filetests/isa/aarch64/vhigh-bits-float.clif create mode 100644 cranelift/filetests/filetests/isa/riscv64/select-spectre-vec.clif create mode 100644 cranelift/filetests/filetests/isa/x64/vany-true-i8.clif diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle index ba17125a0708..ebcca3d3275f 100644 --- a/cranelift/codegen/src/isa/aarch64/lower.isle +++ b/cranelift/codegen/src/isa/aarch64/lower.isle @@ -1444,6 +1444,11 @@ (rule 8 (lower (band (ty_vec128 ty) (bnot _ y) x)) (bic_vec x y (vector_size ty))) +(rule 9 (lower (band (fits_in_64 (ty_scalar_float ty)) x (bnot _ y))) + (bic_vec x y (float_vector_size_in_64 ty))) +(rule 10 (lower (band (fits_in_64 (ty_scalar_float ty)) (bnot _ y) x)) + (bic_vec x y (float_vector_size_in_64 ty))) + ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (rule bor_fits_in_64 -4 (lower (bor (fits_in_64 (ty_int ty)) x y)) @@ -2860,7 +2865,7 @@ ;; in https://github.com/bytecodealliance/wasmtime/issues/2296, ;; although they are not used here. -(rule (lower (vhigh_bits _ vec @ (value_type $I8X16))) +(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 8 16)))) (let ( ;; Replicate the MSB of each of the 16 byte lanes across ;; the whole lane (sshr is an arithmetic right shift). @@ -2884,7 +2889,7 @@ (result Reg (addv zipped (VectorSize.Size16x8)))) (mov_from_vec result 0 (ScalarSize.Size16)))) -(rule (lower (vhigh_bits _ vec @ (value_type $I16X8))) +(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 16 8)))) (let ( ;; Replicate the MSB of each of the 8 16-bit lanes across ;; the whole lane (sshr is an arithmetic right shift). @@ -2897,7 +2902,7 @@ (result Reg (addv anded (VectorSize.Size16x8)))) (mov_from_vec result 0 (ScalarSize.Size16)))) -(rule (lower (vhigh_bits _ vec @ (value_type $I32X4))) +(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 32 4)))) (let ( ;; Replicate the MSB of each of the 4 32-bit lanes across ;; the whole lane (sshr is an arithmetic right shift). @@ -2910,7 +2915,7 @@ (result Reg (addv anded (VectorSize.Size32x4)))) (mov_from_vec result 0 (ScalarSize.Size32)))) -(rule (lower (vhigh_bits _ vec @ (value_type $I64X2))) +(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 64 2)))) (let ( ;; Grab the MSB out of each of the lanes, right-shift to ;; LSB, and add with a left-shift of upper lane's MSB back diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle index f3b91b0767f7..c58b4df83335 100644 --- a/cranelift/codegen/src/isa/riscv64/lower.isle +++ b/cranelift/codegen/src/isa/riscv64/lower.isle @@ -2702,6 +2702,12 @@ (rule 3 (lower (select_spectre_guard (fits_in_64 _) cmp x (i64_from_iconst 0))) (rv_and x (gen_bmask cmp))) +(rule 4 (lower (select_spectre_guard (ty_supported_vec ty) cmp x y)) + (let ((mask XReg (gen_bmask cmp))) + (rv_vor_vv (rv_vand_vx x mask (unmasked) ty) + (rv_vandn_vx y mask (unmasked) ty) + (unmasked) ty))) + ;;;;; Rules for `bmask`;;;;;;;;; (rule (lower (bmask oty x)) diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index d1744469c855..24c1d7ded82b 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -4282,6 +4282,11 @@ ;; This rule zeroes out the upper bits of the XMM register; we need this to ;; avoid undefined bits in scaler_to_vector. (decl bitcast_gpr_to_xmm (u8 Gpr) Xmm) +(rule 1 (bitcast_gpr_to_xmm 8 src) + (if-let true (has_sse41)) + (x64_pinsrb (xmm_zero $I8X16) src 0)) +(rule 0 (bitcast_gpr_to_xmm 8 src) + (x64_pinsrw (xmm_zero $I16X8) (x64_movzbl_rm src) 0)) (rule (bitcast_gpr_to_xmm 16 src) (x64_pinsrw (xmm_zero $I16X8) src 0)) (rule (bitcast_gpr_to_xmm 32 src) diff --git a/cranelift/filetests/filetests/isa/aarch64/vhigh-bits-float.clif b/cranelift/filetests/filetests/isa/aarch64/vhigh-bits-float.clif new file mode 100644 index 000000000000..789e67ad600b --- /dev/null +++ b/cranelift/filetests/filetests/isa/aarch64/vhigh-bits-float.clif @@ -0,0 +1,121 @@ +test compile precise-output +set unwind_info=false +target aarch64 + +function %vhigh_bits_f32x4(f32x4) -> i32 { +block0(v0: f32x4): + v1 = vhigh_bits.i32 v0 + return v1 +} + +; VCode: +; block0: +; sshr v2.4s, v0.4s, #31 +; ldr q4, [const(0)] +; and v6.16b, v2.16b, v4.16b +; addv s16, v6.4s +; mov w0, v16.s[0] +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; sshr v2.4s, v0.4s, #0x1f +; ldr q4, #0x20 +; and v6.16b, v2.16b, v4.16b +; addv s16, v6.4s +; mov w0, v16.s[0] +; ret +; udf #0 +; udf #0 +; udf #1 +; udf #2 +; udf #4 +; udf #8 + +function %vhigh_bits_f64x2(f64x2) -> i32 { +block0(v0: f64x2): + v1 = vhigh_bits.i32 v0 + return v1 +} + +; VCode: +; block0: +; mov x2, v0.d[1] +; mov x4, v0.d[0] +; lsr x6, x2, #63 +; lsr x8, x4, #63 +; add x0, x8, x6, LSL 1 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; mov x2, v0.d[1] +; mov x4, v0.d[0] +; lsr x6, x2, #0x3f +; lsr x8, x4, #0x3f +; add x0, x8, x6, lsl #1 +; ret + +function %vhigh_bits_f16x8(f16x8) -> i32 { +block0(v0: f16x8): + v1 = vhigh_bits.i32 v0 + return v1 +} + +; VCode: +; block0: +; sshr v2.8h, v0.8h, #15 +; ldr q4, [const(0)] +; and v6.16b, v2.16b, v4.16b +; addv h16, v6.8h +; umov w0, v16.h[0] +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; sshr v2.8h, v0.8h, #0xf +; ldr q4, #0x20 +; and v6.16b, v2.16b, v4.16b +; addv h16, v6.8h +; umov w0, v16.h[0] +; ret +; udf #0 +; udf #0 +; .byte 0x01, 0x00, 0x02, 0x00 +; .byte 0x04, 0x00, 0x08, 0x00 +; .byte 0x10, 0x00, 0x20, 0x00 +; .byte 0x40, 0x00, 0x80, 0x00 + +function %band_not_f64(f64, f64) -> f64 { +block0(v0: f64, v1: f64): + v2 = bnot v1 + v3 = band v0, v2 + return v3 +} + +; VCode: +; block0: +; bic v0.8b, v0.8b, v1.8b +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; bic v0.8b, v0.8b, v1.8b +; ret + +function %band_not_f32(f32, f32) -> f32 { +block0(v0: f32, v1: f32): + v2 = bnot v1 + v3 = band v0, v2 + return v3 +} + +; VCode: +; block0: +; bic v0.8b, v0.8b, v1.8b +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; bic v0.8b, v0.8b, v1.8b +; ret diff --git a/cranelift/filetests/filetests/isa/riscv64/select-spectre-vec.clif b/cranelift/filetests/filetests/isa/riscv64/select-spectre-vec.clif new file mode 100644 index 000000000000..04421f964cec --- /dev/null +++ b/cranelift/filetests/filetests/isa/riscv64/select-spectre-vec.clif @@ -0,0 +1,94 @@ +test compile precise-output +set unwind_info=false +target riscv64 has_v + +function %select_spectre_i8x16(i64, i8x16, i8x16) { + fn0 = %callee(i8x16) +block0(v1: i64, v2: i8x16, v3: i8x16): + v4 = select_spectre_guard.i8x16 v1, v2, v3 + call fn0(v4) + return +} + +; VCode: +; addi sp,sp,-16 +; sd ra,8(sp) +; sd fp,0(sp) +; mv fp,sp +; addi sp,sp,-16 +; block0: +; vle8.v v9,-32(incoming_arg) #avl=16, #vtype=(e8, m1, ta, ma) +; vle8.v v8,-16(incoming_arg) #avl=16, #vtype=(e8, m1, ta, ma) +; sltu a0,zero,a0 +; sub a0,zero,a0 +; vand.vx v11,v9,a0 #avl=16, #vtype=(e8, m1, ta, ma) +; vandn.vx v13,v8,a0 #avl=16, #vtype=(e8, m1, ta, ma) +; vor.vv v15,v11,v13 #avl=16, #vtype=(e8, m1, ta, ma) +; vse8.v v15,0(sp) #avl=16, #vtype=(e8, m1, ta, ma) +; load_ext_name_far a2,%callee+0 +; callind a2 +; addi sp,sp,16 +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; addi sp, sp, -0x10 +; sd ra, 8(sp) +; sd s0, 0(sp) +; mv s0, sp +; addi sp, sp, -0x10 +; block1: ; offset 0x14 +; .byte 0x57, 0x70, 0x08, 0xcc +; addi t6, sp, 0x20 +; .byte 0x87, 0x84, 0x0f, 0x02 +; addi t6, sp, 0x30 +; .byte 0x07, 0x84, 0x0f, 0x02 +; snez a0, a0 +; neg a0, a0 +; .byte 0xd7, 0x45, 0x95, 0x26 +; .byte 0xd7, 0x46, 0x85, 0x06 +; .byte 0xd7, 0x87, 0xb6, 0x2a +; .byte 0xa7, 0x07, 0x01, 0x02 +; auipc a2, 0 +; ld a2, 0xc(a2) +; j 0xc +; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee 0 +; .byte 0x00, 0x00, 0x00, 0x00 +; jalr a2 +; addi sp, sp, 0x10 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret + +function %callee(i8x16) { +block0(v0: i8x16): + return +} + +; VCode: +; addi sp,sp,-16 +; sd ra,8(sp) +; sd fp,0(sp) +; mv fp,sp +; block0: +; ld ra,8(sp) +; ld fp,0(sp) +; addi sp,sp,16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; addi sp, sp, -0x10 +; sd ra, 8(sp) +; sd s0, 0(sp) +; mv s0, sp +; block1: ; offset 0x10 +; ld ra, 8(sp) +; ld s0, 0(sp) +; addi sp, sp, 0x10 +; ret + diff --git a/cranelift/filetests/filetests/isa/x64/vany-true-i8.clif b/cranelift/filetests/filetests/isa/x64/vany-true-i8.clif new file mode 100644 index 000000000000..05ea79b2ed0a --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/vany-true-i8.clif @@ -0,0 +1,36 @@ +test compile precise-output +set unwind_info=false +target x86_64 has_sse3 has_ssse3 has_sse41 + +function %vany_true_i8(i8) -> i8 { +block0(v0: i8): + v1 = scalar_to_vector.i8x16 v0 + v2 = vany_true v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; uninit %xmm5 +; pxor %xmm5, %xmm5 +; pinsrb $0x0, %edi, %xmm5 +; ptest %xmm5, %xmm5 +; setne %al +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +; pxor %xmm5, %xmm5 +; pinsrb $0, %edi, %xmm5 +; ptest %xmm5, %xmm5 +; setne %al +; movq %rbp, %rsp +; popq %rbp +; retq diff --git a/cranelift/filetests/filetests/runtests/simd-vhighbits-float.clif b/cranelift/filetests/filetests/runtests/simd-vhighbits-float.clif index b84ae2ef8523..b2b97a0ce87d 100644 --- a/cranelift/filetests/filetests/runtests/simd-vhighbits-float.clif +++ b/cranelift/filetests/filetests/runtests/simd-vhighbits-float.clif @@ -1,5 +1,6 @@ test interpret test run +target aarch64 target s390x target x86_64 has_sse3 has_ssse3 has_sse41 target x86_64 has_sse3 has_ssse3 has_sse41 has_avx