gcc/config/aarch64/aarch64-simd-builtins.def

   1 /* Machine description for AArch64 architecture.
   2    Copyright (C) 2012-2018 Free Software Foundation, Inc.
   3    Contributed by ARM Ltd.
   4
   5    This file is part of GCC.
   6
   7    GCC is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    GCC is distributed in the hope that it will be useful, but
  13    WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with GCC; see the file COPYING3.  If not see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 /* In the list below, the BUILTIN_<ITERATOR> macros expand to create
  22    builtins for each of the modes described by <ITERATOR>.  When adding
  23    new builtins to this list, a helpful idiom to follow is to add
  24    a line for each pattern in the md file.  Thus, ADDP, which has one
  25    pattern defined for the VD_BHSI iterator, and one for DImode, has two
  26    entries below.
  27
  28    Parameter 1 is the 'type' of the intrinsic.  This is used to
  29    describe the type modifiers (for example; unsigned) applied to
  30    each of the parameters to the intrinsic function.
  31
  32    Parameter 2 is the name of the intrinsic.  This is appended
  33    to `__builtin_aarch64_<name><mode>` to give the intrinsic name
  34    as exported to the front-ends.
  35
  36    Parameter 3 describes how to map from the name to the CODE_FOR_
  37    macro holding the RTL pattern for the intrinsic.  This mapping is:
  38    0 - CODE_FOR_aarch64_<name><mode>
  39    1-9 - CODE_FOR_<name><mode><1-9>
  40    10 - CODE_FOR_<name><mode>.  */
  41
  42   BUILTIN_VDC (COMBINE, combine, 0)
  43   VAR1 (COMBINEP, combine, 0, di)
  44   BUILTIN_VB (BINOP, pmul, 0)
  45   BUILTIN_VHSDF_HSDF (BINOP, fmulx, 0)
  46   BUILTIN_VHSDF_DF (UNOP, sqrt, 2)
  47   BUILTIN_VD_BHSI (BINOP, addp, 0)
  48   VAR1 (UNOP, addp, 0, di)
  49   BUILTIN_VDQ_BHSI (UNOP, clrsb, 2)
  50   BUILTIN_VDQ_BHSI (UNOP, clz, 2)
  51   BUILTIN_VS (UNOP, ctz, 2)
  52   BUILTIN_VB (UNOP, popcount, 2)
  53
  54   /* Implemented by aarch64_<sur>q<r>shl<mode>.  */
  55   BUILTIN_VSDQ_I (BINOP, sqshl, 0)
  56   BUILTIN_VSDQ_I (BINOP_UUS, uqshl, 0)
  57   BUILTIN_VSDQ_I (BINOP, sqrshl, 0)
  58   BUILTIN_VSDQ_I (BINOP_UUS, uqrshl, 0)
  59   /* Implemented by aarch64_<su_optab><optab><mode>.  */
  60   BUILTIN_VSDQ_I (BINOP, sqadd, 0)
  61   BUILTIN_VSDQ_I (BINOPU, uqadd, 0)
  62   BUILTIN_VSDQ_I (BINOP, sqsub, 0)
  63   BUILTIN_VSDQ_I (BINOPU, uqsub, 0)
  64   /* Implemented by aarch64_<sur>qadd<mode>.  */
  65   BUILTIN_VSDQ_I (BINOP_SSU, suqadd, 0)
  66   BUILTIN_VSDQ_I (BINOP_UUS, usqadd, 0)
  67
  68   /* Implemented by aarch64_get_dreg<VSTRUCT:mode><VDC:mode>.  */
  69   BUILTIN_VDC (GETREG, get_dregoi, 0)
  70   BUILTIN_VDC (GETREG, get_dregci, 0)
  71   BUILTIN_VDC (GETREG, get_dregxi, 0)
  72   VAR1 (GETREGP, get_dregoi, 0, di)
  73   VAR1 (GETREGP, get_dregci, 0, di)
  74   VAR1 (GETREGP, get_dregxi, 0, di)
  75   /* Implemented by aarch64_get_qreg<VSTRUCT:mode><VQ:mode>.  */
  76   BUILTIN_VQ (GETREG, get_qregoi, 0)
  77   BUILTIN_VQ (GETREG, get_qregci, 0)
  78   BUILTIN_VQ (GETREG, get_qregxi, 0)
  79   VAR1 (GETREGP, get_qregoi, 0, v2di)
  80   VAR1 (GETREGP, get_qregci, 0, v2di)
  81   VAR1 (GETREGP, get_qregxi, 0, v2di)
  82   /* Implemented by aarch64_set_qreg<VSTRUCT:mode><VQ:mode>.  */
  83   BUILTIN_VQ (SETREG, set_qregoi, 0)
  84   BUILTIN_VQ (SETREG, set_qregci, 0)
  85   BUILTIN_VQ (SETREG, set_qregxi, 0)
  86   VAR1 (SETREGP, set_qregoi, 0, v2di)
  87   VAR1 (SETREGP, set_qregci, 0, v2di)
  88   VAR1 (SETREGP, set_qregxi, 0, v2di)
  89   /* Implemented by aarch64_ld1x2<VQ:mode>. */
  90   BUILTIN_VQ (LOADSTRUCT, ld1x2, 0)
  91   /* Implemented by aarch64_ld1x2<VDC:mode>. */
  92   BUILTIN_VDC (LOADSTRUCT, ld1x2, 0)
  93   /* Implemented by aarch64_ld<VSTRUCT:nregs><VDC:mode>.  */
  94   BUILTIN_VDC (LOADSTRUCT, ld2, 0)
  95   BUILTIN_VDC (LOADSTRUCT, ld3, 0)
  96   BUILTIN_VDC (LOADSTRUCT, ld4, 0)
  97   /* Implemented by aarch64_ld<VSTRUCT:nregs><VQ:mode>.  */
  98   BUILTIN_VQ (LOADSTRUCT, ld2, 0)
  99   BUILTIN_VQ (LOADSTRUCT, ld3, 0)
 100   BUILTIN_VQ (LOADSTRUCT, ld4, 0)
 101   /* Implemented by aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>.  */
 102   BUILTIN_VALLDIF (LOADSTRUCT, ld2r, 0)
 103   BUILTIN_VALLDIF (LOADSTRUCT, ld3r, 0)
 104   BUILTIN_VALLDIF (LOADSTRUCT, ld4r, 0)
 105   /* Implemented by aarch64_ld<VSTRUCT:nregs>_lane<VQ:mode>.  */
 106   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld2_lane, 0)
 107   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld3_lane, 0)
 108   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld4_lane, 0)
 109   /* Implemented by aarch64_st<VSTRUCT:nregs><VDC:mode>.  */
 110   BUILTIN_VDC (STORESTRUCT, st2, 0)
 111   BUILTIN_VDC (STORESTRUCT, st3, 0)
 112   BUILTIN_VDC (STORESTRUCT, st4, 0)
 113   /* Implemented by aarch64_st<VSTRUCT:nregs><VQ:mode>.  */
 114   BUILTIN_VQ (STORESTRUCT, st2, 0)
 115   BUILTIN_VQ (STORESTRUCT, st3, 0)
 116   BUILTIN_VQ (STORESTRUCT, st4, 0)
 117
 118   BUILTIN_VALLDIF (STORESTRUCT_LANE, st2_lane, 0)
 119   BUILTIN_VALLDIF (STORESTRUCT_LANE, st3_lane, 0)
 120   BUILTIN_VALLDIF (STORESTRUCT_LANE, st4_lane, 0)
 121
 122   BUILTIN_VQW (BINOP, saddl2, 0)
 123   BUILTIN_VQW (BINOP, uaddl2, 0)
 124   BUILTIN_VQW (BINOP, ssubl2, 0)
 125   BUILTIN_VQW (BINOP, usubl2, 0)
 126   BUILTIN_VQW (BINOP, saddw2, 0)
 127   BUILTIN_VQW (BINOP, uaddw2, 0)
 128   BUILTIN_VQW (BINOP, ssubw2, 0)
 129   BUILTIN_VQW (BINOP, usubw2, 0)
 130   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>.  */
 131   BUILTIN_VD_BHSI (BINOP, saddl, 0)
 132   BUILTIN_VD_BHSI (BINOP, uaddl, 0)
 133   BUILTIN_VD_BHSI (BINOP, ssubl, 0)
 134   BUILTIN_VD_BHSI (BINOP, usubl, 0)
 135   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>.  */
 136   BUILTIN_VD_BHSI (BINOP, saddw, 0)
 137   BUILTIN_VD_BHSI (BINOP, uaddw, 0)
 138   BUILTIN_VD_BHSI (BINOP, ssubw, 0)
 139   BUILTIN_VD_BHSI (BINOP, usubw, 0)
 140   /* Implemented by aarch64_<sur>h<addsub><mode>.  */
 141   BUILTIN_VDQ_BHSI (BINOP, shadd, 0)
 142   BUILTIN_VDQ_BHSI (BINOP, shsub, 0)
 143   BUILTIN_VDQ_BHSI (BINOP, uhadd, 0)
 144   BUILTIN_VDQ_BHSI (BINOP, uhsub, 0)
 145   BUILTIN_VDQ_BHSI (BINOP, srhadd, 0)
 146   BUILTIN_VDQ_BHSI (BINOP, urhadd, 0)
 147   /* Implemented by aarch64_<sur><addsub>hn<mode>.  */
 148   BUILTIN_VQN (BINOP, addhn, 0)
 149   BUILTIN_VQN (BINOP, subhn, 0)
 150   BUILTIN_VQN (BINOP, raddhn, 0)
 151   BUILTIN_VQN (BINOP, rsubhn, 0)
 152   /* Implemented by aarch64_<sur><addsub>hn2<mode>.  */
 153   BUILTIN_VQN (TERNOP, addhn2, 0)
 154   BUILTIN_VQN (TERNOP, subhn2, 0)
 155   BUILTIN_VQN (TERNOP, raddhn2, 0)
 156   BUILTIN_VQN (TERNOP, rsubhn2, 0)
 157
 158   BUILTIN_VSQN_HSDI (UNOP, sqmovun, 0)
 159   /* Implemented by aarch64_<sur>qmovn<mode>.  */
 160   BUILTIN_VSQN_HSDI (UNOP, sqmovn, 0)
 161   BUILTIN_VSQN_HSDI (UNOP, uqmovn, 0)
 162   /* Implemented by aarch64_s<optab><mode>.  */
 163   BUILTIN_VSDQ_I (UNOP, sqabs, 0)
 164   BUILTIN_VSDQ_I (UNOP, sqneg, 0)
 165
 166   /* Implemented by aarch64_sqdml<SBINQOPS:as>l<mode>.  */
 167   BUILTIN_VSD_HSI (TERNOP, sqdmlal, 0)
 168   BUILTIN_VSD_HSI (TERNOP, sqdmlsl, 0)
 169   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_lane<mode>.  */
 170   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_lane, 0)
 171   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_lane, 0)
 172   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_laneq<mode>.  */
 173   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_laneq, 0)
 174   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_laneq, 0)
 175   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_n<mode>.  */
 176   BUILTIN_VD_HSI (TERNOP, sqdmlal_n, 0)
 177   BUILTIN_VD_HSI (TERNOP, sqdmlsl_n, 0)
 178
 179   BUILTIN_VQ_HSI (TERNOP, sqdmlal2, 0)
 180   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2, 0)
 181   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_lane, 0)
 182   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_lane, 0)
 183   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_laneq, 0)
 184   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_laneq, 0)
 185   BUILTIN_VQ_HSI (TERNOP, sqdmlal2_n, 0)
 186   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2_n, 0)
 187
 188   BUILTIN_VSD_HSI (BINOP, sqdmull, 0)
 189   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_lane, 0)
 190   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_laneq, 0)
 191   BUILTIN_VD_HSI (BINOP, sqdmull_n, 0)
 192   BUILTIN_VQ_HSI (BINOP, sqdmull2, 0)
 193   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_lane, 0)
 194   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_laneq, 0)
 195   BUILTIN_VQ_HSI (BINOP, sqdmull2_n, 0)
 196   /* Implemented by aarch64_sq<r>dmulh<mode>.  */
 197   BUILTIN_VSDQ_HSI (BINOP, sqdmulh, 0)
 198   BUILTIN_VSDQ_HSI (BINOP, sqrdmulh, 0)
 199   /* Implemented by aarch64_sq<r>dmulh_lane<q><mode>.  */
 200   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_lane, 0)
 201   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_laneq, 0)
 202   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_lane, 0)
 203   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_laneq, 0)
 204
 205   BUILTIN_VSDQ_I_DI (BINOP, ashl, 3)
 206   /* Implemented by aarch64_<sur>shl<mode>.  */
 207   BUILTIN_VSDQ_I_DI (BINOP, sshl, 0)
 208   BUILTIN_VSDQ_I_DI (BINOP_UUS, ushl, 0)
 209   BUILTIN_VSDQ_I_DI (BINOP, srshl, 0)
 210   BUILTIN_VSDQ_I_DI (BINOP_UUS, urshl, 0)
 211
 212   /* Implemented by aarch64_<sur><dotprod>{_lane}{q}<dot_mode>.  */
 213   BUILTIN_VB (TERNOP, sdot, 0)
 214   BUILTIN_VB (TERNOPU, udot, 0)
 215   BUILTIN_VB (QUADOP_LANE, sdot_lane, 0)
 216   BUILTIN_VB (QUADOPU_LANE, udot_lane, 0)
 217   BUILTIN_VB (QUADOP_LANE, sdot_laneq, 0)
 218   BUILTIN_VB (QUADOPU_LANE, udot_laneq, 0)
 219
 220   BUILTIN_VDQ_I (SHIFTIMM, ashr, 3)
 221   VAR1 (SHIFTIMM, ashr_simd, 0, di)
 222   BUILTIN_VDQ_I (SHIFTIMM, lshr, 3)
 223   VAR1 (USHIFTIMM, lshr_simd, 0, di)
 224   /* Implemented by aarch64_<sur>shr_n<mode>.  */
 225   BUILTIN_VSDQ_I_DI (SHIFTIMM, srshr_n, 0)
 226   BUILTIN_VSDQ_I_DI (USHIFTIMM, urshr_n, 0)
 227   /* Implemented by aarch64_<sur>sra_n<mode>.  */
 228   BUILTIN_VSDQ_I_DI (SHIFTACC, ssra_n, 0)
 229   BUILTIN_VSDQ_I_DI (USHIFTACC, usra_n, 0)
 230   BUILTIN_VSDQ_I_DI (SHIFTACC, srsra_n, 0)
 231   BUILTIN_VSDQ_I_DI (USHIFTACC, ursra_n, 0)
 232   /* Implemented by aarch64_<sur>shll_n<mode>.  */
 233   BUILTIN_VD_BHSI (SHIFTIMM, sshll_n, 0)
 234   BUILTIN_VD_BHSI (USHIFTIMM, ushll_n, 0)
 235   /* Implemented by aarch64_<sur>shll2_n<mode>.  */
 236   BUILTIN_VQW (SHIFTIMM, sshll2_n, 0)
 237   BUILTIN_VQW (SHIFTIMM, ushll2_n, 0)
 238   /* Implemented by aarch64_<sur>q<r>shr<u>n_n<mode>.  */
 239   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrun_n, 0)
 240   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrun_n, 0)
 241   BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrn_n, 0)
 242   BUILTIN_VSQN_HSDI (USHIFTIMM, uqshrn_n, 0)
 243   BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrn_n, 0)
 244   BUILTIN_VSQN_HSDI (USHIFTIMM, uqrshrn_n, 0)
 245   /* Implemented by aarch64_<sur>s<lr>i_n<mode>.  */
 246   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssri_n, 0)
 247   BUILTIN_VSDQ_I_DI (USHIFTACC, usri_n, 0)
 248   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssli_n, 0)
 249   VAR2 (SHIFTINSERTP, ssli_n, 0, di, v2di)
 250   BUILTIN_VSDQ_I_DI (USHIFTACC, usli_n, 0)
 251   /* Implemented by aarch64_<sur>qshl<u>_n<mode>.  */
 252   BUILTIN_VSDQ_I (SHIFTIMM_USS, sqshlu_n, 0)
 253   BUILTIN_VSDQ_I (SHIFTIMM, sqshl_n, 0)
 254   BUILTIN_VSDQ_I (USHIFTIMM, uqshl_n, 0)
 255
 256   /* Implemented by aarch64_reduc_plus_<mode>.  */
 257   BUILTIN_VALL (UNOP, reduc_plus_scal_, 10)
 258
 259   /* Implemented by reduc_<maxmin_uns>_scal_<mode> (producing scalar).  */
 260   BUILTIN_VDQIF_F16 (UNOP, reduc_smax_scal_, 10)
 261   BUILTIN_VDQIF_F16 (UNOP, reduc_smin_scal_, 10)
 262   BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10)
 263   BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10)
 264   BUILTIN_VHSDF (UNOP, reduc_smax_nan_scal_, 10)
 265   BUILTIN_VHSDF (UNOP, reduc_smin_nan_scal_, 10)
 266
 267   /* Implemented by <maxmin_uns><mode>3.
 268      smax variants map to fmaxnm,
 269      smax_nan variants map to fmax.  */
 270   BUILTIN_VDQ_BHSI (BINOP, smax, 3)
 271   BUILTIN_VDQ_BHSI (BINOP, smin, 3)
 272   BUILTIN_VDQ_BHSI (BINOP, umax, 3)
 273   BUILTIN_VDQ_BHSI (BINOP, umin, 3)
 274   BUILTIN_VHSDF_DF (BINOP, smax_nan, 3)
 275   BUILTIN_VHSDF_DF (BINOP, smin_nan, 3)
 276
 277   /* Implemented by <maxmin_uns><mode>3.  */
 278   BUILTIN_VHSDF_HSDF (BINOP, fmax, 3)
 279   BUILTIN_VHSDF_HSDF (BINOP, fmin, 3)
 280
 281   /* Implemented by aarch64_<maxmin_uns>p<mode>.  */
 282   BUILTIN_VDQ_BHSI (BINOP, smaxp, 0)
 283   BUILTIN_VDQ_BHSI (BINOP, sminp, 0)
 284   BUILTIN_VDQ_BHSI (BINOP, umaxp, 0)
 285   BUILTIN_VDQ_BHSI (BINOP, uminp, 0)
 286   BUILTIN_VHSDF (BINOP, smaxp, 0)
 287   BUILTIN_VHSDF (BINOP, sminp, 0)
 288   BUILTIN_VHSDF (BINOP, smax_nanp, 0)
 289   BUILTIN_VHSDF (BINOP, smin_nanp, 0)
 290
 291   /* Implemented by <frint_pattern><mode>2.  */
 292   BUILTIN_VHSDF (UNOP, btrunc, 2)
 293   BUILTIN_VHSDF (UNOP, ceil, 2)
 294   BUILTIN_VHSDF (UNOP, floor, 2)
 295   BUILTIN_VHSDF (UNOP, nearbyint, 2)
 296   BUILTIN_VHSDF (UNOP, rint, 2)
 297   BUILTIN_VHSDF (UNOP, round, 2)
 298   BUILTIN_VHSDF_DF (UNOP, frintn, 2)
 299
 300   VAR1 (UNOP, btrunc, 2, hf)
 301   VAR1 (UNOP, ceil, 2, hf)
 302   VAR1 (UNOP, floor, 2, hf)
 303   VAR1 (UNOP, frintn, 2, hf)
 304   VAR1 (UNOP, nearbyint, 2, hf)
 305   VAR1 (UNOP, rint, 2, hf)
 306   VAR1 (UNOP, round, 2, hf)
 307
 308   /* Implemented by l<fcvt_pattern><su_optab><VQDF:mode><vcvt_target>2.  */
 309   VAR1 (UNOP, lbtruncv4hf, 2, v4hi)
 310   VAR1 (UNOP, lbtruncv8hf, 2, v8hi)
 311   VAR1 (UNOP, lbtruncv2sf, 2, v2si)
 312   VAR1 (UNOP, lbtruncv4sf, 2, v4si)
 313   VAR1 (UNOP, lbtruncv2df, 2, v2di)
 314
 315   VAR1 (UNOPUS, lbtruncuv4hf, 2, v4hi)
 316   VAR1 (UNOPUS, lbtruncuv8hf, 2, v8hi)
 317   VAR1 (UNOPUS, lbtruncuv2sf, 2, v2si)
 318   VAR1 (UNOPUS, lbtruncuv4sf, 2, v4si)
 319   VAR1 (UNOPUS, lbtruncuv2df, 2, v2di)
 320
 321   VAR1 (UNOP, lroundv4hf, 2, v4hi)
 322   VAR1 (UNOP, lroundv8hf, 2, v8hi)
 323   VAR1 (UNOP, lroundv2sf, 2, v2si)
 324   VAR1 (UNOP, lroundv4sf, 2, v4si)
 325   VAR1 (UNOP, lroundv2df, 2, v2di)
 326   /* Implemented by l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2.  */
 327   BUILTIN_GPI_I16 (UNOP, lroundhf, 2)
 328   VAR1 (UNOP, lroundsf, 2, si)
 329   VAR1 (UNOP, lrounddf, 2, di)
 330
 331   VAR1 (UNOPUS, lrounduv4hf, 2, v4hi)
 332   VAR1 (UNOPUS, lrounduv8hf, 2, v8hi)
 333   VAR1 (UNOPUS, lrounduv2sf, 2, v2si)
 334   VAR1 (UNOPUS, lrounduv4sf, 2, v4si)
 335   VAR1 (UNOPUS, lrounduv2df, 2, v2di)
 336   BUILTIN_GPI_I16 (UNOPUS, lrounduhf, 2)
 337   VAR1 (UNOPUS, lroundusf, 2, si)
 338   VAR1 (UNOPUS, lroundudf, 2, di)
 339
 340   VAR1 (UNOP, lceilv4hf, 2, v4hi)
 341   VAR1 (UNOP, lceilv8hf, 2, v8hi)
 342   VAR1 (UNOP, lceilv2sf, 2, v2si)
 343   VAR1 (UNOP, lceilv4sf, 2, v4si)
 344   VAR1 (UNOP, lceilv2df, 2, v2di)
 345   BUILTIN_GPI_I16 (UNOP, lceilhf, 2)
 346
 347   VAR1 (UNOPUS, lceiluv4hf, 2, v4hi)
 348   VAR1 (UNOPUS, lceiluv8hf, 2, v8hi)
 349   VAR1 (UNOPUS, lceiluv2sf, 2, v2si)
 350   VAR1 (UNOPUS, lceiluv4sf, 2, v4si)
 351   VAR1 (UNOPUS, lceiluv2df, 2, v2di)
 352   BUILTIN_GPI_I16 (UNOPUS, lceiluhf, 2)
 353   VAR1 (UNOPUS, lceilusf, 2, si)
 354   VAR1 (UNOPUS, lceiludf, 2, di)
 355
 356   VAR1 (UNOP, lfloorv4hf, 2, v4hi)
 357   VAR1 (UNOP, lfloorv8hf, 2, v8hi)
 358   VAR1 (UNOP, lfloorv2sf, 2, v2si)
 359   VAR1 (UNOP, lfloorv4sf, 2, v4si)
 360   VAR1 (UNOP, lfloorv2df, 2, v2di)
 361   BUILTIN_GPI_I16 (UNOP, lfloorhf, 2)
 362
 363   VAR1 (UNOPUS, lflooruv4hf, 2, v4hi)
 364   VAR1 (UNOPUS, lflooruv8hf, 2, v8hi)
 365   VAR1 (UNOPUS, lflooruv2sf, 2, v2si)
 366   VAR1 (UNOPUS, lflooruv4sf, 2, v4si)
 367   VAR1 (UNOPUS, lflooruv2df, 2, v2di)
 368   BUILTIN_GPI_I16 (UNOPUS, lflooruhf, 2)
 369   VAR1 (UNOPUS, lfloorusf, 2, si)
 370   VAR1 (UNOPUS, lfloorudf, 2, di)
 371
 372   VAR1 (UNOP, lfrintnv4hf, 2, v4hi)
 373   VAR1 (UNOP, lfrintnv8hf, 2, v8hi)
 374   VAR1 (UNOP, lfrintnv2sf, 2, v2si)
 375   VAR1 (UNOP, lfrintnv4sf, 2, v4si)
 376   VAR1 (UNOP, lfrintnv2df, 2, v2di)
 377   BUILTIN_GPI_I16 (UNOP, lfrintnhf, 2)
 378   VAR1 (UNOP, lfrintnsf, 2, si)
 379   VAR1 (UNOP, lfrintndf, 2, di)
 380
 381   VAR1 (UNOPUS, lfrintnuv4hf, 2, v4hi)
 382   VAR1 (UNOPUS, lfrintnuv8hf, 2, v8hi)
 383   VAR1 (UNOPUS, lfrintnuv2sf, 2, v2si)
 384   VAR1 (UNOPUS, lfrintnuv4sf, 2, v4si)
 385   VAR1 (UNOPUS, lfrintnuv2df, 2, v2di)
 386   BUILTIN_GPI_I16 (UNOPUS, lfrintnuhf, 2)
 387   VAR1 (UNOPUS, lfrintnusf, 2, si)
 388   VAR1 (UNOPUS, lfrintnudf, 2, di)
 389
 390   /* Implemented by <optab><fcvt_target><VDQF:mode>2.  */
 391   VAR1 (UNOP, floatv4hi, 2, v4hf)
 392   VAR1 (UNOP, floatv8hi, 2, v8hf)
 393   VAR1 (UNOP, floatv2si, 2, v2sf)
 394   VAR1 (UNOP, floatv4si, 2, v4sf)
 395   VAR1 (UNOP, floatv2di, 2, v2df)
 396
 397   VAR1 (UNOP, floatunsv4hi, 2, v4hf)
 398   VAR1 (UNOP, floatunsv8hi, 2, v8hf)
 399   VAR1 (UNOP, floatunsv2si, 2, v2sf)
 400   VAR1 (UNOP, floatunsv4si, 2, v4sf)
 401   VAR1 (UNOP, floatunsv2di, 2, v2df)
 402
 403   VAR5 (UNOPU, bswap, 2, v4hi, v8hi, v2si, v4si, v2di)
 404
 405   BUILTIN_VB (UNOP, rbit, 0)
 406
 407   /* Implemented by
 408      aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>.  */
 409   BUILTIN_VALL (BINOP, zip1, 0)
 410   BUILTIN_VALL (BINOP, zip2, 0)
 411   BUILTIN_VALL (BINOP, uzp1, 0)
 412   BUILTIN_VALL (BINOP, uzp2, 0)
 413   BUILTIN_VALL (BINOP, trn1, 0)
 414   BUILTIN_VALL (BINOP, trn2, 0)
 415
 416   /* Implemented by
 417      aarch64_frecp<FRECP:frecp_suffix><mode>.  */
 418   BUILTIN_GPF_F16 (UNOP, frecpe, 0)
 419   BUILTIN_GPF_F16 (UNOP, frecpx, 0)
 420
 421   BUILTIN_VDQ_SI (UNOP, urecpe, 0)
 422
 423   BUILTIN_VHSDF (UNOP, frecpe, 0)
 424   BUILTIN_VHSDF_HSDF (BINOP, frecps, 0)
 425
 426   /* Implemented by a mixture of abs2 patterns.  Note the DImode builtin is
 427      only ever used for the int64x1_t intrinsic, there is no scalar version.  */
 428   BUILTIN_VSDQ_I_DI (UNOP, abs, 0)
 429   BUILTIN_VHSDF (UNOP, abs, 2)
 430   VAR1 (UNOP, abs, 2, hf)
 431
 432   BUILTIN_VQ_HSF (UNOP, vec_unpacks_hi_, 10)
 433   VAR1 (BINOP, float_truncate_hi_, 0, v4sf)
 434   VAR1 (BINOP, float_truncate_hi_, 0, v8hf)
 435
 436   VAR1 (UNOP, float_extend_lo_, 0, v2df)
 437   VAR1 (UNOP, float_extend_lo_,  0, v4sf)
 438   BUILTIN_VDF (UNOP, float_truncate_lo_, 0)
 439
 440   /* Implemented by aarch64_ld1<VALL_F16:mode>.  */
 441   BUILTIN_VALL_F16 (LOAD1, ld1, 0)
 442   VAR1(STORE1P, ld1, 0, v2di)
 443
 444   /* Implemented by aarch64_st1<VALL_F16:mode>.  */
 445   BUILTIN_VALL_F16 (STORE1, st1, 0)
 446   VAR1(STORE1P, st1, 0, v2di)
 447
 448   /* Implemented by aarch64_ld1x3<VALLDIF:mode>.  */
 449   BUILTIN_VALLDIF (LOADSTRUCT, ld1x3, 0)
 450
 451   /* Implemented by aarch64_ld1x4<VALLDIF:mode>.  */
 452   BUILTIN_VALLDIF (LOADSTRUCT, ld1x4, 0)
 453
 454   /* Implemented by aarch64_st1x2<VALLDIF:mode>.  */
 455   BUILTIN_VALLDIF (STORESTRUCT, st1x2, 0)
 456
 457   /* Implemented by aarch64_st1x3<VALLDIF:mode>.  */
 458   BUILTIN_VALLDIF (STORESTRUCT, st1x3, 0)
 459
 460   /* Implemented by aarch64_st1x4<VALLDIF:mode>.  */
 461   BUILTIN_VALLDIF (STORESTRUCT, st1x4, 0)
 462
 463   /* Implemented by fma<mode>4.  */
 464   BUILTIN_VHSDF (TERNOP, fma, 4)
 465   VAR1 (TERNOP, fma, 4, hf)
 466   /* Implemented by fnma<mode>4.  */
 467   BUILTIN_VHSDF (TERNOP, fnma, 4)
 468   VAR1 (TERNOP, fnma, 4, hf)
 469
 470   /* Implemented by aarch64_simd_bsl<mode>.  */
 471   BUILTIN_VDQQH (BSL_P, simd_bsl, 0)
 472   VAR2 (BSL_P, simd_bsl,0, di, v2di)
 473   BUILTIN_VSDQ_I_DI (BSL_U, simd_bsl, 0)
 474   BUILTIN_VALLDIF (BSL_S, simd_bsl, 0)
 475
 476   /* Implemented by aarch64_crypto_aes<op><mode>.  */
 477   VAR1 (BINOPU, crypto_aese, 0, v16qi)
 478   VAR1 (BINOPU, crypto_aesd, 0, v16qi)
 479   VAR1 (UNOPU, crypto_aesmc, 0, v16qi)
 480   VAR1 (UNOPU, crypto_aesimc, 0, v16qi)
 481
 482   /* Implemented by aarch64_crypto_sha1<op><mode>.  */
 483   VAR1 (UNOPU, crypto_sha1h, 0, si)
 484   VAR1 (BINOPU, crypto_sha1su1, 0, v4si)
 485   VAR1 (TERNOPU, crypto_sha1c, 0, v4si)
 486   VAR1 (TERNOPU, crypto_sha1m, 0, v4si)
 487   VAR1 (TERNOPU, crypto_sha1p, 0, v4si)
 488   VAR1 (TERNOPU, crypto_sha1su0, 0, v4si)
 489
 490   /* Implemented by aarch64_crypto_sha256<op><mode>.  */
 491   VAR1 (TERNOPU, crypto_sha256h, 0, v4si)
 492   VAR1 (TERNOPU, crypto_sha256h2, 0, v4si)
 493   VAR1 (BINOPU, crypto_sha256su0, 0, v4si)
 494   VAR1 (TERNOPU, crypto_sha256su1, 0, v4si)
 495
 496   /* Implemented by aarch64_crypto_pmull<mode>.  */
 497   VAR1 (BINOPP, crypto_pmull, 0, di)
 498   VAR1 (BINOPP, crypto_pmull, 0, v2di)
 499
 500   /* Implemented by aarch64_tbl3<mode>.  */
 501   VAR1 (BINOP, tbl3, 0, v8qi)
 502   VAR1 (BINOP, tbl3, 0, v16qi)
 503
 504   /* Implemented by aarch64_qtbl3<mode>.  */
 505   VAR1 (BINOP, qtbl3, 0, v8qi)
 506   VAR1 (BINOP, qtbl3, 0, v16qi)
 507
 508   /* Implemented by aarch64_qtbl4<mode>.  */
 509   VAR1 (BINOP, qtbl4, 0, v8qi)
 510   VAR1 (BINOP, qtbl4, 0, v16qi)
 511
 512   /* Implemented by aarch64_tbx4<mode>.  */
 513   VAR1 (TERNOP, tbx4, 0, v8qi)
 514   VAR1 (TERNOP, tbx4, 0, v16qi)
 515
 516   /* Implemented by aarch64_qtbx3<mode>.  */
 517   VAR1 (TERNOP, qtbx3, 0, v8qi)
 518   VAR1 (TERNOP, qtbx3, 0, v16qi)
 519
 520   /* Implemented by aarch64_qtbx4<mode>.  */
 521   VAR1 (TERNOP, qtbx4, 0, v8qi)
 522   VAR1 (TERNOP, qtbx4, 0, v16qi)
 523
 524   /* Builtins for ARMv8.1-A Adv.SIMD instructions.  */
 525
 526   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>.  */
 527   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlah, 0)
 528   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlsh, 0)
 529
 530   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>.  */
 531   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_lane, 0)
 532   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_lane, 0)
 533
 534   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>.  */
 535   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_laneq, 0)
 536   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_laneq, 0)
 537
 538   /* Implemented by <FCVT_F2FIXED/FIXED2F:fcvt_fixed_insn><*><*>3.  */
 539   BUILTIN_VSDQ_HSDI (SHIFTIMM, scvtf, 3)
 540   BUILTIN_VSDQ_HSDI (FCVTIMM_SUS, ucvtf, 3)
 541   BUILTIN_VHSDF_HSDF (SHIFTIMM, fcvtzs, 3)
 542   BUILTIN_VHSDF_HSDF (SHIFTIMM_USS, fcvtzu, 3)
 543   VAR1 (SHIFTIMM, scvtfsi, 3, hf)
 544   VAR1 (SHIFTIMM, scvtfdi, 3, hf)
 545   VAR1 (FCVTIMM_SUS, ucvtfsi, 3, hf)
 546   VAR1 (FCVTIMM_SUS, ucvtfdi, 3, hf)
 547   BUILTIN_GPI (SHIFTIMM, fcvtzshf, 3)
 548   BUILTIN_GPI (SHIFTIMM_USS, fcvtzuhf, 3)
 549
 550   /* Implemented by aarch64_rsqrte<mode>.  */
 551   BUILTIN_VHSDF_HSDF (UNOP, rsqrte, 0)
 552
 553   /* Implemented by aarch64_rsqrts<mode>.  */
 554   BUILTIN_VHSDF_HSDF (BINOP, rsqrts, 0)
 555
 556   /* Implemented by fabd<mode>3.  */
 557   BUILTIN_VHSDF_HSDF (BINOP, fabd, 3)
 558
 559   /* Implemented by aarch64_faddp<mode>.  */
 560   BUILTIN_VHSDF (BINOP, faddp, 0)
 561
 562   /* Implemented by aarch64_cm<optab><mode>.  */
 563   BUILTIN_VHSDF_HSDF (BINOP_USS, cmeq, 0)
 564   BUILTIN_VHSDF_HSDF (BINOP_USS, cmge, 0)
 565   BUILTIN_VHSDF_HSDF (BINOP_USS, cmgt, 0)
 566   BUILTIN_VHSDF_HSDF (BINOP_USS, cmle, 0)
 567   BUILTIN_VHSDF_HSDF (BINOP_USS, cmlt, 0)
 568
 569   /* Implemented by neg<mode>2.  */
 570   BUILTIN_VHSDF_HSDF (UNOP, neg, 2)
 571
 572   /* Implemented by aarch64_fac<optab><mode>.  */
 573   BUILTIN_VHSDF_HSDF (BINOP_USS, faclt, 0)
 574   BUILTIN_VHSDF_HSDF (BINOP_USS, facle, 0)
 575   BUILTIN_VHSDF_HSDF (BINOP_USS, facgt, 0)
 576   BUILTIN_VHSDF_HSDF (BINOP_USS, facge, 0)
 577
 578   /* Implemented by sqrt<mode>2.  */
 579   VAR1 (UNOP, sqrt, 2, hf)
 580
 581   /* Implemented by <optab><mode>hf2.  */
 582   VAR1 (UNOP, floatdi, 2, hf)
 583   VAR1 (UNOP, floatsi, 2, hf)
 584   VAR1 (UNOP, floathi, 2, hf)
 585   VAR1 (UNOPUS, floatunsdi, 2, hf)
 586   VAR1 (UNOPUS, floatunssi, 2, hf)
 587   VAR1 (UNOPUS, floatunshi, 2, hf)
 588   BUILTIN_GPI_I16 (UNOP, fix_trunchf, 2)
 589   BUILTIN_GPI (UNOP, fix_truncsf, 2)
 590   BUILTIN_GPI (UNOP, fix_truncdf, 2)
 591   BUILTIN_GPI_I16 (UNOPUS, fixuns_trunchf, 2)
 592   BUILTIN_GPI (UNOPUS, fixuns_truncsf, 2)
 593   BUILTIN_GPI (UNOPUS, fixuns_truncdf, 2)
 594
 595   /* Implemented by aarch64_sm3ss1qv4si.  */
 596   VAR1 (TERNOPU, sm3ss1q, 0, v4si)
 597   /* Implemented by aarch64_sm3tt<sm3tt_op>qv4si.  */
 598   VAR1 (QUADOPUI, sm3tt1aq, 0, v4si)
 599   VAR1 (QUADOPUI, sm3tt1bq, 0, v4si)
 600   VAR1 (QUADOPUI, sm3tt2aq, 0, v4si)
 601   VAR1 (QUADOPUI, sm3tt2bq, 0, v4si)
 602   /* Implemented by aarch64_sm3partw<sm3part_op>qv4si.  */
 603   VAR1 (TERNOPU, sm3partw1q, 0, v4si)
 604   VAR1 (TERNOPU, sm3partw2q, 0, v4si)
 605   /* Implemented by aarch64_sm4eqv4si.  */
 606   VAR1 (BINOPU, sm4eq, 0, v4si)
 607   /* Implemented by aarch64_sm4ekeyqv4si.  */
 608   VAR1 (BINOPU, sm4ekeyq, 0, v4si)
 609   /* Implemented by aarch64_crypto_sha512hqv2di.  */
 610   VAR1 (TERNOPU, crypto_sha512hq, 0, v2di)
 611   /* Implemented by aarch64_sha512h2qv2di.  */
 612   VAR1 (TERNOPU, crypto_sha512h2q, 0, v2di)
 613   /* Implemented by aarch64_crypto_sha512su0qv2di.  */
 614   VAR1 (BINOPU, crypto_sha512su0q, 0, v2di)
 615   /* Implemented by aarch64_crypto_sha512su1qv2di.  */
 616   VAR1 (TERNOPU, crypto_sha512su1q, 0, v2di)
 617   /* Implemented by aarch64_eor3qv8hi.  */
 618   VAR1 (TERNOPU, eor3q, 0, v8hi)
 619   /* Implemented by aarch64_rax1qv2di.  */
 620   VAR1 (BINOPU, rax1q, 0, v2di)
 621   /* Implemented by aarch64_xarqv2di.  */
 622   VAR1 (TERNOPUI, xarq, 0, v2di)
 623   /* Implemented by aarch64_bcaxqv8hi.  */
 624   VAR1 (TERNOPU, bcaxq, 0, v8hi)
 625
 626   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_low<mode>.  */
 627   VAR1 (TERNOP, fmlal_low, 0, v2sf)
 628   VAR1 (TERNOP, fmlsl_low, 0, v2sf)
 629   VAR1 (TERNOP, fmlalq_low, 0, v4sf)
 630   VAR1 (TERNOP, fmlslq_low, 0, v4sf)
 631   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_high<mode>.  */
 632   VAR1 (TERNOP, fmlal_high, 0, v2sf)
 633   VAR1 (TERNOP, fmlsl_high, 0, v2sf)
 634   VAR1 (TERNOP, fmlalq_high, 0, v4sf)
 635   VAR1 (TERNOP, fmlslq_high, 0, v4sf)
 636   /* Implemented by aarch64_fml<f16mac1>l_lane_lowv2sf.  */
 637   VAR1 (QUADOP_LANE, fmlal_lane_low, 0, v2sf)
 638   VAR1 (QUADOP_LANE, fmlsl_lane_low, 0, v2sf)
 639   /* Implemented by aarch64_fml<f16mac1>l_laneq_lowv2sf.  */
 640   VAR1 (QUADOP_LANE, fmlal_laneq_low, 0, v2sf)
 641   VAR1 (QUADOP_LANE, fmlsl_laneq_low, 0, v2sf)
 642   /* Implemented by aarch64_fml<f16mac1>lq_lane_lowv4sf.  */
 643   VAR1 (QUADOP_LANE, fmlalq_lane_low, 0, v4sf)
 644   VAR1 (QUADOP_LANE, fmlslq_lane_low, 0, v4sf)
 645   /* Implemented by aarch64_fml<f16mac1>lq_laneq_lowv4sf.  */
 646   VAR1 (QUADOP_LANE, fmlalq_laneq_low, 0, v4sf)
 647   VAR1 (QUADOP_LANE, fmlslq_laneq_low, 0, v4sf)
 648   /* Implemented by aarch64_fml<f16mac1>l_lane_highv2sf.  */
 649   VAR1 (QUADOP_LANE, fmlal_lane_high, 0, v2sf)
 650   VAR1 (QUADOP_LANE, fmlsl_lane_high, 0, v2sf)
 651   /* Implemented by aarch64_fml<f16mac1>l_laneq_highv2sf.  */
 652   VAR1 (QUADOP_LANE, fmlal_laneq_high, 0, v2sf)
 653   VAR1 (QUADOP_LANE, fmlsl_laneq_high, 0, v2sf)
 654   /* Implemented by aarch64_fml<f16mac1>lq_lane_highv4sf.  */
 655   VAR1 (QUADOP_LANE, fmlalq_lane_high, 0, v4sf)
 656   VAR1 (QUADOP_LANE, fmlslq_lane_high, 0, v4sf)
 657   /* Implemented by aarch64_fml<f16mac1>lq_laneq_highv4sf.  */
 658   VAR1 (QUADOP_LANE, fmlalq_laneq_high, 0, v4sf)
 659   VAR1 (QUADOP_LANE, fmlslq_laneq_high, 0, v4sf)