[AArch64][SVE] Add intrinsics for FFR manipulation

Summary:
Implements the following intrinsics:
  - llvm.aarch64.sve.setffr
  - llvm.aarch64.sve.rdffr
  - llvm.aarch64.sve.rdffr.z
  - llvm.aarch64.sve.wrffr

Reviewers: sdesmalen, efriedma, dancgr, rengolin

Reviewed By: efriedma

Subscribers: tschuett, kristof.beyls, hiraditya, rkruppe, psnobl, cameron.mcinally, cfe-commits, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D73097

llvm/include/llvm/IR/IntrinsicsAArch64.td

@@ -1290,6 +1290,15 @@ def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
 
 def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
 
+//
+// FFR manipulation
+//
+
+def int_aarch64_sve_rdffr   : GCCBuiltin<"__builtin_sve_svrdffr">,   Intrinsic<[llvm_nxv16i1_ty], []>;
+def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>;
+def int_aarch64_sve_setffr  : GCCBuiltin<"__builtin_sve_svsetffr">,  Intrinsic<[], []>;
+def int_aarch64_sve_wrffr   : GCCBuiltin<"__builtin_sve_svwrffr">,   Intrinsic<[], [llvm_nxv16i1_ty]>;
+
 //
 // Saturating scalar arithmetic
 //

llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td

@@ -98,11 +98,11 @@ def AArch64ptest     : SDNode<"AArch64ISD::PTEST", SDT_AArch64PTest>;
 
 let Predicates = [HasSVE] in {
 
-  def RDFFR_PPz  : sve_int_rdffr_pred<0b0, "rdffr">;
+  defm RDFFR_PPz  : sve_int_rdffr_pred<0b0, "rdffr", int_aarch64_sve_rdffr_z>;
   def  RDFFRS_PPz : sve_int_rdffr_pred<0b1, "rdffrs">;
-  def RDFFR_P    : sve_int_rdffr_unpred<"rdffr">;
+  defm RDFFR_P    : sve_int_rdffr_unpred<"rdffr", int_aarch64_sve_rdffr>;
-  def SETFFR     : sve_int_setffr<"setffr">;
+  def  SETFFR     : sve_int_setffr<"setffr", int_aarch64_sve_setffr>;
-  def WRFFR      : sve_int_wrffr<"wrffr">;
+  def  WRFFR      : sve_int_wrffr<"wrffr", int_aarch64_sve_wrffr>;
 
   defm ADD_ZZZ   : sve_int_bin_cons_arit_0<0b000, "add", add>;
   defm SUB_ZZZ   : sve_int_bin_cons_arit_0<0b001, "sub", sub>;

llvm/lib/Target/AArch64/SVEInstrFormats.td

@@ -5105,6 +5105,17 @@ class sve_int_rdffr_pred<bit s, string asm>
   let Uses = [FFR];
 }
 
+multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> {
+  def _REAL : sve_int_rdffr_pred<s, asm>;
+
+  // We need a layer of indirection because early machine code passes balk at
+  // physical register (i.e. FFR) uses that have no previous definition.
+  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
+  def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>,
+           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>;
+  }
+}
+
 class sve_int_rdffr_unpred<string asm> : I<
   (outs PPR8:$Pd), (ins),
   asm, "\t$Pd",
@@ -5117,11 +5128,22 @@ class sve_int_rdffr_unpred<string asm> : I<
   let Uses = [FFR];
 }
 
-class sve_int_wrffr<string asm>
+multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> {
+  def _REAL : sve_int_rdffr_unpred<asm>;
+
+  // We need a layer of indirection because early machine code passes balk at
+  // physical register (i.e. FFR) uses that have no previous definition.
+  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
+  def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>,
+           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>;
+  }
+}
+
+class sve_int_wrffr<string asm, SDPatternOperator op>
 : I<(outs), (ins PPR8:$Pn),
   asm, "\t$Pn",
   "",
-  []>, Sched<[]> {
+  [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> {
   bits<4> Pn;
   let Inst{31-9} = 0b00100101001010001001000;
   let Inst{8-5}  = Pn;
@@ -5131,11 +5153,11 @@ class sve_int_wrffr<string asm>
   let Defs = [FFR];
 }
 
-class sve_int_setffr<string asm>
+class sve_int_setffr<string asm, SDPatternOperator op>
 : I<(outs), (ins),
   asm, "",
   "",
-  []>, Sched<[]> {
+  [(op)]>, Sched<[]> {
   let Inst{31-0} = 0b00100101001011001001000000000000;
 
   let hasSideEffects = 1;

llvm/test/CodeGen/AArch64/sve-intrinsics-ffr-manipulation.ll

@@ -0,0 +1,50 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; RDFFR
+;
+
+define <vscale x 16 x i1> @rdffr() {
+; CHECK-LABEL: rdffr:
+; CHECK: rdffr p0.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
+  ret <vscale x 16 x i1> %out
+}
+
+define <vscale x 16 x i1> @rdffr_z(<vscale x 16 x i1> %pg) {
+; CHECK-LABEL: rdffr_z:
+; CHECK: rdffr p0.b, p0/z
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1> %pg)
+  ret <vscale x 16 x i1> %out
+}
+
+;
+; SETFFR
+;
+
+define void @set_ffr() {
+; CHECK-LABEL: set_ffr:
+; CHECK: setffr
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.setffr()
+  ret void
+}
+
+;
+; WRFFR
+;
+
+define void @wrffr(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: wrffr:
+; CHECK: wrffr p0.b
+; CHECK-NEXT: ret
+  call void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1> %a)
+  ret void
+}
+
+declare <vscale x 16 x i1> @llvm.aarch64.sve.rdffr()
+declare <vscale x 16 x i1> @llvm.aarch64.sve.rdffr.z(<vscale x 16 x i1>)
+declare void @llvm.aarch64.sve.setffr()
+declare void @llvm.aarch64.sve.wrffr(<vscale x 16 x i1>)