cranelift: fix missing ISLE lowering rules from #12197 on AArch64, x86_64, and RISC-V

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -1444,6 +1444,11 @@
 (rule 8 (lower (band (ty_vec128 ty) (bnot _ y) x))
       (bic_vec x y (vector_size ty)))
 
+(rule 9 (lower (band (fits_in_64 (ty_scalar_float ty)) x (bnot _ y)))
+      (bic_vec x y (float_vector_size_in_64 ty)))
+(rule 10 (lower (band (fits_in_64 (ty_scalar_float ty)) (bnot _ y) x))
+      (bic_vec x y (float_vector_size_in_64 ty)))
+
 ;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule bor_fits_in_64 -4 (lower (bor (fits_in_64 (ty_int ty)) x y))
@@ -2860,7 +2865,7 @@
 ;; in https://github.com/bytecodealliance/wasmtime/issues/2296,
 ;; although they are not used here.
 
-(rule (lower (vhigh_bits _ vec @ (value_type $I8X16)))
+(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 8 16))))
       (let (
             ;; Replicate the MSB of each of the 16 byte lanes across
             ;; the whole lane (sshr is an arithmetic right shift).
@@ -2884,7 +2889,7 @@
             (result Reg (addv zipped (VectorSize.Size16x8))))
         (mov_from_vec result 0 (ScalarSize.Size16))))
 
-(rule (lower (vhigh_bits _ vec @ (value_type $I16X8)))
+(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 16 8))))
       (let (
             ;; Replicate the MSB of each of the 8 16-bit lanes across
             ;; the whole lane (sshr is an arithmetic right shift).
@@ -2897,7 +2902,7 @@
             (result Reg (addv anded (VectorSize.Size16x8))))
         (mov_from_vec result 0 (ScalarSize.Size16))))
 
-(rule (lower (vhigh_bits _ vec @ (value_type $I32X4)))
+(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 32 4))))
       (let (
             ;; Replicate the MSB of each of the 4 32-bit lanes across
             ;; the whole lane (sshr is an arithmetic right shift).
@@ -2910,7 +2915,7 @@
             (result Reg (addv anded (VectorSize.Size32x4))))
         (mov_from_vec result 0 (ScalarSize.Size32))))
 
-(rule (lower (vhigh_bits _ vec @ (value_type $I64X2)))
+(rule (lower (vhigh_bits _ vec @ (value_type (multi_lane 64 2))))
       (let (
             ;; Grab the MSB out of each of the lanes, right-shift to
             ;; LSB, and add with a left-shift of upper lane's MSB back

cranelift/codegen/src/isa/riscv64/lower.isle

@@ -2702,6 +2702,12 @@
 (rule 3 (lower (select_spectre_guard (fits_in_64 _) cmp x (i64_from_iconst 0)))
   (rv_and x (gen_bmask cmp)))
 
+(rule 4 (lower (select_spectre_guard (ty_supported_vec ty) cmp x y))
+  (let ((mask XReg (gen_bmask cmp)))
+    (rv_vor_vv (rv_vand_vx x mask (unmasked) ty)
+               (rv_vandn_vx y mask (unmasked) ty)
+               (unmasked) ty)))
+
 ;;;;;  Rules for `bmask`;;;;;;;;;
 (rule
   (lower (bmask oty x))

cranelift/codegen/src/isa/x64/inst.isle

@@ -4282,6 +4282,11 @@
 ;; This rule zeroes out the upper bits of the XMM register; we need this to
 ;; avoid undefined bits in scaler_to_vector.
 (decl bitcast_gpr_to_xmm (u8 Gpr) Xmm)
+(rule 1 (bitcast_gpr_to_xmm 8 src)
+      (if-let true (has_sse41))
+      (x64_pinsrb (xmm_zero $I8X16) src 0))
+(rule 0 (bitcast_gpr_to_xmm 8 src)
+      (x64_pinsrw (xmm_zero $I16X8) (x64_movzbl_rm src) 0))
 (rule (bitcast_gpr_to_xmm 16 src)
       (x64_pinsrw (xmm_zero $I16X8) src 0))
 (rule (bitcast_gpr_to_xmm 32 src)

cranelift/filetests/filetests/isa/aarch64/vhigh-bits-float.clif

@@ -0,0 +1,121 @@
+test compile precise-output
+set unwind_info=false
+target aarch64
+
+function %vhigh_bits_f32x4(f32x4) -> i32 {
+block0(v0: f32x4):
+    v1 = vhigh_bits.i32 v0
+    return v1
+}
+
+; VCode:
+; block0:
+;   sshr v2.4s, v0.4s, #31
+;   ldr q4, [const(0)]
+;   and v6.16b, v2.16b, v4.16b
+;   addv s16, v6.4s
+;   mov w0, v16.s[0]
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   sshr v2.4s, v0.4s, #0x1f
+;   ldr q4, #0x20
+;   and v6.16b, v2.16b, v4.16b
+;   addv s16, v6.4s
+;   mov w0, v16.s[0]
+;   ret
+;   udf #0
+;   udf #0
+;   udf #1
+;   udf #2
+;   udf #4
+;   udf #8
+
+function %vhigh_bits_f64x2(f64x2) -> i32 {
+block0(v0: f64x2):
+    v1 = vhigh_bits.i32 v0
+    return v1
+}
+
+; VCode:
+; block0:
+;   mov x2, v0.d[1]
+;   mov x4, v0.d[0]
+;   lsr x6, x2, #63
+;   lsr x8, x4, #63
+;   add x0, x8, x6, LSL 1
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   mov x2, v0.d[1]
+;   mov x4, v0.d[0]
+;   lsr x6, x2, #0x3f
+;   lsr x8, x4, #0x3f
+;   add x0, x8, x6, lsl #1
+;   ret
+
+function %vhigh_bits_f16x8(f16x8) -> i32 {
+block0(v0: f16x8):
+    v1 = vhigh_bits.i32 v0
+    return v1
+}
+
+; VCode:
+; block0:
+;   sshr v2.8h, v0.8h, #15
+;   ldr q4, [const(0)]
+;   and v6.16b, v2.16b, v4.16b
+;   addv h16, v6.8h
+;   umov w0, v16.h[0]
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   sshr v2.8h, v0.8h, #0xf
+;   ldr q4, #0x20
+;   and v6.16b, v2.16b, v4.16b
+;   addv h16, v6.8h
+;   umov w0, v16.h[0]
+;   ret
+;   udf #0
+;   udf #0
+;   .byte 0x01, 0x00, 0x02, 0x00
+;   .byte 0x04, 0x00, 0x08, 0x00
+;   .byte 0x10, 0x00, 0x20, 0x00
+;   .byte 0x40, 0x00, 0x80, 0x00
+
+function %band_not_f64(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+    v2 = bnot v1
+    v3 = band v0, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   bic v0.8b, v0.8b, v1.8b
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   bic v0.8b, v0.8b, v1.8b
+;   ret
+
+function %band_not_f32(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+    v2 = bnot v1
+    v3 = band v0, v2
+    return v3
+}
+
+; VCode:
+; block0:
+;   bic v0.8b, v0.8b, v1.8b
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   bic v0.8b, v0.8b, v1.8b
+;   ret

cranelift/filetests/filetests/isa/riscv64/select-spectre-vec.clif

@@ -0,0 +1,94 @@
+test compile precise-output
+set unwind_info=false
+target riscv64 has_v
+
+function %select_spectre_i8x16(i64, i8x16, i8x16) {
+    fn0 = %callee(i8x16)
+block0(v1: i64, v2: i8x16, v3: i8x16):
+    v4 = select_spectre_guard.i8x16 v1, v2, v3
+    call fn0(v4)
+    return
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+;   addi sp,sp,-16
+; block0:
+;   vle8.v v9,-32(incoming_arg) #avl=16, #vtype=(e8, m1, ta, ma)
+;   vle8.v v8,-16(incoming_arg) #avl=16, #vtype=(e8, m1, ta, ma)
+;   sltu a0,zero,a0
+;   sub a0,zero,a0
+;   vand.vx v11,v9,a0 #avl=16, #vtype=(e8, m1, ta, ma)
+;   vandn.vx v13,v8,a0 #avl=16, #vtype=(e8, m1, ta, ma)
+;   vor.vv v15,v11,v13 #avl=16, #vtype=(e8, m1, ta, ma)
+;   vse8.v v15,0(sp) #avl=16, #vtype=(e8, m1, ta, ma)
+;   load_ext_name_far a2,%callee+0
+;   callind a2
+;   addi sp,sp,16
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+;   addi sp, sp, -0x10
+; block1: ; offset 0x14
+;   .byte 0x57, 0x70, 0x08, 0xcc
+;   addi t6, sp, 0x20
+;   .byte 0x87, 0x84, 0x0f, 0x02
+;   addi t6, sp, 0x30
+;   .byte 0x07, 0x84, 0x0f, 0x02
+;   snez a0, a0
+;   neg a0, a0
+;   .byte 0xd7, 0x45, 0x95, 0x26
+;   .byte 0xd7, 0x46, 0x85, 0x06
+;   .byte 0xd7, 0x87, 0xb6, 0x2a
+;   .byte 0xa7, 0x07, 0x01, 0x02
+;   auipc a2, 0
+;   ld a2, 0xc(a2)
+;   j 0xc
+;   .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee 0
+;   .byte 0x00, 0x00, 0x00, 0x00
+;   jalr a2
+;   addi sp, sp, 0x10
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
+function %callee(i8x16) {
+block0(v0: i8x16):
+    return
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+; block0:
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+; block1: ; offset 0x10
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+

cranelift/filetests/filetests/isa/x64/vany-true-i8.clif

@@ -0,0 +1,36 @@
+test compile precise-output
+set unwind_info=false
+target x86_64 has_sse3 has_ssse3 has_sse41
+
+function %vany_true_i8(i8) -> i8 {
+block0(v0: i8):
+    v1 = scalar_to_vector.i8x16 v0
+    v2 = vany_true v1
+    return v2
+}
+
+; VCode:
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block0:
+;   uninit  %xmm5
+;   pxor %xmm5, %xmm5
+;   pinsrb $0x0, %edi, %xmm5
+;   ptest %xmm5, %xmm5
+;   setne %al
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   pxor %xmm5, %xmm5
+;   pinsrb $0, %edi, %xmm5
+;   ptest %xmm5, %xmm5
+;   setne %al
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq

cranelift/filetests/filetests/runtests/simd-vhighbits-float.clif

@@ -1,5 +1,6 @@
 test interpret
 test run
+target aarch64
 target s390x
 target x86_64 has_sse3 has_ssse3 has_sse41
 target x86_64 has_sse3 has_ssse3 has_sse41 has_avx