gcc/config/rs6000/vector.md

   1 ;; Expander definitions for vector support between altivec & vsx.  No
   2 ;; instructions are in this file, this file provides the generic vector
   3 ;; expander, and the actual vector instructions will be in altivec.md and
   4 ;; vsx.md
   5
   6 ;; Copyright (C) 2009-2023 Free Software Foundation, Inc.
   7 ;; Contributed by Michael Meissner <meissner@linux.vnet.ibm.com>
   8
   9 ;; This file is part of GCC.
  10
  11 ;; GCC is free software; you can redistribute it and/or modify it
  12 ;; under the terms of the GNU General Public License as published
  13 ;; by the Free Software Foundation; either version 3, or (at your
  14 ;; option) any later version.
  15
  16 ;; GCC is distributed in the hope that it will be useful, but WITHOUT
  17 ;; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  18 ;; or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  19 ;; License for more details.
  20
  21 ;; You should have received a copy of the GNU General Public License
  22 ;; along with GCC; see the file COPYING3.  If not see
  23 ;; <http://www.gnu.org/licenses/>.
  24
  25
  26 ;; Vector int modes
  27 (define_mode_iterator VEC_I [V16QI V8HI V4SI V2DI])
  28
  29 ;; Vector int modes for comparison, shift and rotation
  30 (define_mode_iterator VEC_IC [V16QI V8HI V4SI V2DI (V1TI "TARGET_POWER10")])
  31
  32 ;; 128-bit int modes
  33 (define_mode_iterator VEC_TI [V1TI TI])
  34
  35 ;; Vector int modes for parity
  36 (define_mode_iterator VEC_IP [V8HI
  37                               V4SI
  38                               V2DI
  39                               V1TI
  40                               TI])
  41
  42 ;; Vector float modes
  43 (define_mode_iterator VEC_F [V4SF V2DF])
  44
  45 ;; Vector arithmetic modes
  46 (define_mode_iterator VEC_A [V16QI V8HI V4SI V2DI V4SF V2DF])
  47
  48 ;; Vector modes that need alginment via permutes
  49 (define_mode_iterator VEC_K [V16QI V8HI V4SI V4SF])
  50
  51 ;; Vector logical modes
  52 (define_mode_iterator VEC_L [V16QI V8HI V4SI V2DI V4SF V2DF V1TI TI KF TF])
  53
  54 ;; Vector modes for moves.  Don't do TImode or TFmode here, since their
  55 ;; moves are handled elsewhere.
  56 (define_mode_iterator VEC_M [V16QI V8HI V4SI V2DI V4SF V2DF V1TI KF])
  57
  58 ;; Vector modes for types that don't need a realignment under VSX
  59 (define_mode_iterator VEC_N [V4SI V4SF V2DI V2DF V1TI KF TF])
  60
  61 ;; Vector comparison modes
  62 (define_mode_iterator VEC_C [V16QI V8HI V4SI V2DI V4SF V2DF V1TI])
  63
  64 ;; Vector init/extract modes
  65 (define_mode_iterator VEC_E [V16QI
  66                              V8HI
  67                              V4SI
  68                              V2DI
  69                              V4SF
  70                              V2DF
  71                              (V32QI     "TARGET_MMA")
  72                              (V16HI     "TARGET_MMA")
  73                              (V8SI      "TARGET_MMA")
  74                              (V4DI      "TARGET_MMA")
  75                              (V8SF      "TARGET_MMA")
  76                              (V4DF      "TARGET_MMA")])
  77
  78 ;; Vector modes for 64-bit base types
  79 (define_mode_iterator VEC_64 [V2DI V2DF])
  80
  81 ;; Vector integer modes
  82 (define_mode_iterator VI [V4SI V8HI V16QI])
  83
  84 ;; Base type from vector mode
  85 (define_mode_attr VEC_base [(V32QI "QI")
  86                             (V16QI "QI")
  87                             (V16HI "HI")
  88                             (V8HI  "HI")
  89                             (V8SI  "SI")
  90                             (V4SI  "SI")
  91                             (V4DI  "DI")
  92                             (V2DI  "DI")
  93                             (V8SF  "SF")
  94                             (V4SF  "SF")
  95                             (V4DF  "DF")
  96                             (V2DF  "DF")
  97                             (V1TI  "TI")
  98                             (TI    "TI")])
  99
 100 ;; As above, but in lower case
 101 (define_mode_attr VEC_base_l [(V32QI "qi")
 102                               (V16QI "qi")
 103                               (V16HI "hi")
 104                               (V8HI  "hi")
 105                               (V8SI  "si")
 106                               (V4SI  "si")
 107                               (V4DI  "di")
 108                               (V2DI  "di")
 109                               (V8SF  "sf")
 110                               (V4SF  "sf")
 111                               (V4DF  "df")
 112                               (V2DF  "df")
 113                               (V1TI  "ti")
 114                               (TI    "ti")])
 115
 116 ;; Same size integer type for floating point data
 117 (define_mode_attr VEC_int [(V4SF  "v4si")
 118                            (V2DF  "v2di")])
 119
 120 (define_mode_attr VEC_INT [(V4SF  "V4SI")
 121                            (V2DF  "V2DI")])
 122
 123 ;; constants for unspec
 124 (define_c_enum "unspec" [UNSPEC_PREDICATE
 125                          UNSPEC_REDUC
 126                          UNSPEC_NEZ_P])
 127
 128 ;; Vector reduction code iterators
 129 (define_code_iterator VEC_reduc [plus smin smax])
 130
 131 (define_code_attr VEC_reduc_name [(plus "plus")
 132                                   (smin "smin")
 133                                   (smax "smax")])
 134
 135 (define_code_attr VEC_reduc_rtx [(plus "add")
 136                                  (smin "smin")
 137                                  (smax "smax")])
 138
 139 ;; code iterators and attributes for vector FP comparison operators:
 140 (define_code_iterator
 141   vector_fp_comparison_simple [lt le ne ungt unge unlt unle])
 142 (define_code_iterator
 143   vector_fp_comparison_complex [ltgt uneq unordered ordered])
 144
 145 \f
 146 ;; Vector move instructions.  Little-endian VSX loads and stores require
 147 ;; special handling to circumvent "element endianness."
 148 (define_expand "mov<mode>"
 149   [(set (match_operand:VEC_M 0 "nonimmediate_operand")
 150         (match_operand:VEC_M 1 "any_operand"))]
 151   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 152 {
 153   if (can_create_pseudo_p ())
 154     {
 155       if (CONSTANT_P (operands[1]))
 156         {
 157           if (FLOAT128_VECTOR_P (<MODE>mode))
 158             {
 159               if (!easy_fp_constant (operands[1], <MODE>mode))
 160                 operands[1] = force_const_mem (<MODE>mode, operands[1]);
 161             }
 162           else if (!easy_vector_constant (operands[1], <MODE>mode))
 163             operands[1] = force_const_mem (<MODE>mode, operands[1]);
 164         }
 165
 166       if (!vlogical_operand (operands[0], <MODE>mode)
 167           && !vlogical_operand (operands[1], <MODE>mode))
 168         operands[1] = force_reg (<MODE>mode, operands[1]);
 169     }
 170   /* When generating load/store instructions to/from VSX registers on
 171      pre-power9 hardware in little endian mode, we need to emit register
 172      permute instructions to byte swap the contents, since the VSX load/store
 173      instructions do not include a byte swap as part of their operation.
 174      Altivec loads and stores have no such problem, so we skip them below.  */
 175   if (!BYTES_BIG_ENDIAN
 176       && VECTOR_MEM_VSX_P (<MODE>mode)
 177       && !TARGET_P9_VECTOR
 178       && !gpr_or_gpr_p (operands[0], operands[1])
 179       && ((memory_operand (operands[0], <MODE>mode)
 180            && !altivec_indexed_or_indirect_operand(operands[0], <MODE>mode))
 181           ^ (memory_operand (operands[1], <MODE>mode)
 182              && !altivec_indexed_or_indirect_operand(operands[1], <MODE>mode))))
 183     {
 184       rs6000_emit_le_vsx_move (operands[0], operands[1], <MODE>mode);
 185       DONE;
 186     }
 187 })
 188
 189 ;; Generic vector floating point load/store instructions.  These will match
 190 ;; insns defined in vsx.md or altivec.md depending on the switches.
 191 (define_expand "vector_load_<mode>"
 192   [(set (match_operand:VEC_M 0 "vfloat_operand")
 193         (match_operand:VEC_M 1 "memory_operand"))]
 194   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 195   "")
 196
 197 (define_expand "vector_store_<mode>"
 198   [(set (match_operand:VEC_M 0 "memory_operand")
 199         (match_operand:VEC_M 1 "vfloat_operand"))]
 200   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
 201   "")
 202
 203 ;; Splits if a GPR register was chosen for the move
 204 (define_split
 205   [(set (match_operand:VEC_L 0 "nonimmediate_operand")
 206         (match_operand:VEC_L 1 "input_operand"))]
 207   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)
 208    && reload_completed
 209    && gpr_or_gpr_p (operands[0], operands[1])
 210    && !direct_move_p (operands[0], operands[1])
 211    && !quad_load_store_p (operands[0], operands[1])"
 212   [(pc)]
 213 {
 214   rs6000_split_multireg_move (operands[0], operands[1]);
 215   DONE;
 216 })
 217
 218 \f
 219 ;; Generic floating point vector arithmetic support
 220 (define_expand "add<mode>3"
 221   [(set (match_operand:VEC_F 0 "vfloat_operand")
 222         (plus:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 223                     (match_operand:VEC_F 2 "vfloat_operand")))]
 224   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 225   "")
 226
 227 (define_expand "sub<mode>3"
 228   [(set (match_operand:VEC_F 0 "vfloat_operand")
 229         (minus:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 230                      (match_operand:VEC_F 2 "vfloat_operand")))]
 231   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 232   "")
 233
 234 (define_expand "mul<mode>3"
 235   [(set (match_operand:VEC_F 0 "vfloat_operand")
 236         (mult:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 237                     (match_operand:VEC_F 2 "vfloat_operand")))]
 238   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 239 {
 240   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 241     {
 242       emit_insn (gen_altivec_mulv4sf3 (operands[0], operands[1], operands[2]));
 243       DONE;
 244     }
 245 })
 246
 247 (define_expand "div<mode>3"
 248   [(set (match_operand:VEC_F 0 "vfloat_operand")
 249         (div:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 250                    (match_operand:VEC_F 2 "vfloat_operand")))]
 251   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 252 {
 253   if (RS6000_RECIP_AUTO_RE_P (<MODE>mode)
 254       && can_create_pseudo_p () && flag_finite_math_only
 255       && !flag_trapping_math && flag_reciprocal_math)
 256     {
 257       rs6000_emit_swdiv (operands[0], operands[1], operands[2], true);
 258       DONE;
 259     }
 260 })
 261
 262 (define_expand "neg<mode>2"
 263   [(set (match_operand:VEC_F 0 "vfloat_operand")
 264         (neg:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 265   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 266 {
 267   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 268     {
 269       emit_insn (gen_altivec_negv4sf2 (operands[0], operands[1]));
 270       DONE;
 271     }
 272 })
 273
 274 (define_expand "abs<mode>2"
 275   [(set (match_operand:VEC_F 0 "vfloat_operand")
 276         (abs:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 277   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 278 {
 279   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 280     {
 281       emit_insn (gen_altivec_absv4sf2 (operands[0], operands[1]));
 282       DONE;
 283     }
 284 })
 285
 286 (define_expand "smin<mode>3"
 287   [(set (match_operand:VEC_F 0 "register_operand")
 288         (smin:VEC_F (match_operand:VEC_F 1 "register_operand")
 289                     (match_operand:VEC_F 2 "register_operand")))]
 290   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 291   "")
 292
 293 (define_expand "smax<mode>3"
 294   [(set (match_operand:VEC_F 0 "register_operand")
 295         (smax:VEC_F (match_operand:VEC_F 1 "register_operand")
 296                     (match_operand:VEC_F 2 "register_operand")))]
 297   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 298   "")
 299
 300
 301 (define_expand "sqrt<mode>2"
 302   [(set (match_operand:VEC_F 0 "vfloat_operand")
 303         (sqrt:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 304   "VECTOR_UNIT_VSX_P (<MODE>mode)"
 305 {
 306   if (<MODE>mode == V4SFmode
 307       && !optimize_function_for_size_p (cfun)
 308       && flag_finite_math_only && !flag_trapping_math
 309       && flag_unsafe_math_optimizations)
 310     {
 311       rs6000_emit_swsqrt (operands[0], operands[1], 0);
 312       DONE;
 313     }
 314 })
 315
 316 (define_expand "rsqrte<mode>2"
 317   [(set (match_operand:VEC_F 0 "vfloat_operand")
 318         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 319                       UNSPEC_RSQRT))]
 320   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 321   "")
 322
 323 (define_expand "re<mode>2"
 324   [(set (match_operand:VEC_F 0 "vfloat_operand")
 325         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 326                       UNSPEC_FRES))]
 327   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 328   "")
 329
 330 (define_expand "ftrunc<mode>2"
 331   [(set (match_operand:VEC_F 0 "vfloat_operand")
 332         (fix:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 333   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 334   "")
 335
 336 (define_expand "vector_ceil<mode>2"
 337   [(set (match_operand:VEC_F 0 "vfloat_operand")
 338         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 339                       UNSPEC_FRIP))]
 340   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 341   "")
 342
 343 (define_expand "vector_floor<mode>2"
 344   [(set (match_operand:VEC_F 0 "vfloat_operand")
 345         (unspec:VEC_F [(match_operand:VEC_F 1 "vfloat_operand")]
 346                       UNSPEC_FRIM))]
 347   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 348   "")
 349
 350 (define_expand "vector_btrunc<mode>2"
 351   [(set (match_operand:VEC_F 0 "vfloat_operand")
 352         (fix:VEC_F (match_operand:VEC_F 1 "vfloat_operand")))]
 353   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 354   "")
 355
 356 (define_expand "vector_copysign<mode>3"
 357   [(set (match_operand:VEC_F 0 "vfloat_operand")
 358         (copysign:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
 359                         (match_operand:VEC_F 2 "vfloat_operand")))]
 360   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 361 {
 362   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
 363     {
 364       emit_insn (gen_altivec_copysign_v4sf3 (operands[0], operands[1],
 365                                              operands[2]));
 366       DONE;
 367     }
 368 })
 369
 370 \f
 371 ;; Vector comparisons
 372 (define_expand "vcond<mode><mode>"
 373   [(set (match_operand:VEC_F 0 "vfloat_operand")
 374         (if_then_else:VEC_F
 375          (match_operator 3 "comparison_operator"
 376                          [(match_operand:VEC_F 4 "vfloat_operand")
 377                           (match_operand:VEC_F 5 "vfloat_operand")])
 378          (match_operand:VEC_F 1 "vfloat_operand")
 379          (match_operand:VEC_F 2 "vfloat_operand")))]
 380   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 381 {
 382   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 383                                     operands[3], operands[4], operands[5]))
 384     DONE;
 385   else
 386     gcc_unreachable ();
 387 })
 388
 389 (define_expand "vcond<mode><mode>"
 390   [(set (match_operand:VEC_I 0 "vint_operand")
 391         (if_then_else:VEC_I
 392          (match_operator 3 "comparison_operator"
 393                          [(match_operand:VEC_I 4 "vint_operand")
 394                           (match_operand:VEC_I 5 "vint_operand")])
 395          (match_operand:VEC_I 1 "vector_int_reg_or_same_bit")
 396          (match_operand:VEC_I 2 "vector_int_reg_or_same_bit")))]
 397   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 398 {
 399   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 400                                     operands[3], operands[4], operands[5]))
 401     DONE;
 402   else
 403     gcc_unreachable ();
 404 })
 405
 406 (define_expand "vcondv4sfv4si"
 407   [(set (match_operand:V4SF 0 "vfloat_operand")
 408         (if_then_else:V4SF
 409          (match_operator 3 "comparison_operator"
 410                          [(match_operand:V4SI 4 "vint_operand")
 411                           (match_operand:V4SI 5 "vint_operand")])
 412          (match_operand:V4SF 1 "vfloat_operand")
 413          (match_operand:V4SF 2 "vfloat_operand")))]
 414   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 415    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 416 {
 417   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 418                                     operands[3], operands[4], operands[5]))
 419     DONE;
 420   else
 421     gcc_unreachable ();
 422 })
 423
 424 (define_expand "vcondv4siv4sf"
 425   [(set (match_operand:V4SI 0 "vint_operand")
 426         (if_then_else:V4SI
 427          (match_operator 3 "comparison_operator"
 428                          [(match_operand:V4SF 4 "vfloat_operand")
 429                           (match_operand:V4SF 5 "vfloat_operand")])
 430          (match_operand:V4SI 1 "vint_operand")
 431          (match_operand:V4SI 2 "vint_operand")))]
 432   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 433    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 434 {
 435   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 436                                     operands[3], operands[4], operands[5]))
 437     DONE;
 438   else
 439     gcc_unreachable ();
 440 })
 441
 442 (define_expand "vcondv2dfv2di"
 443   [(set (match_operand:V2DF 0 "vfloat_operand")
 444         (if_then_else:V2DF
 445          (match_operator 3 "comparison_operator"
 446                          [(match_operand:V2DI 4 "vint_operand")
 447                           (match_operand:V2DI 5 "vint_operand")])
 448          (match_operand:V2DF 1 "vfloat_operand")
 449          (match_operand:V2DF 2 "vfloat_operand")))]
 450   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 451    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 452 {
 453   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 454                                     operands[3], operands[4], operands[5]))
 455     DONE;
 456   else
 457     gcc_unreachable ();
 458 })
 459
 460 (define_expand "vcondv2div2df"
 461   [(set (match_operand:V2DI 0 "vint_operand")
 462         (if_then_else:V2DI
 463          (match_operator 3 "comparison_operator"
 464                          [(match_operand:V2DF 4 "vfloat_operand")
 465                           (match_operand:V2DF 5 "vfloat_operand")])
 466          (match_operand:V2DI 1 "vint_operand")
 467          (match_operand:V2DI 2 "vint_operand")))]
 468   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 469    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 470 {
 471   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 472                                     operands[3], operands[4], operands[5]))
 473     DONE;
 474   else
 475     gcc_unreachable ();
 476 })
 477
 478 (define_expand "vcondu<mode><mode>"
 479   [(set (match_operand:VEC_I 0 "vint_operand")
 480         (if_then_else:VEC_I
 481          (match_operator 3 "comparison_operator"
 482                          [(match_operand:VEC_I 4 "vint_operand")
 483                           (match_operand:VEC_I 5 "vint_operand")])
 484          (match_operand:VEC_I 1 "vector_int_reg_or_same_bit")
 485          (match_operand:VEC_I 2 "vector_int_reg_or_same_bit")))]
 486   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 487 {
 488   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 489                                     operands[3], operands[4], operands[5]))
 490     DONE;
 491   else
 492     gcc_unreachable ();
 493 })
 494
 495 (define_expand "vconduv4sfv4si"
 496   [(set (match_operand:V4SF 0 "vfloat_operand")
 497         (if_then_else:V4SF
 498          (match_operator 3 "comparison_operator"
 499                          [(match_operand:V4SI 4 "vint_operand")
 500                           (match_operand:V4SI 5 "vint_operand")])
 501          (match_operand:V4SF 1 "vfloat_operand")
 502          (match_operand:V4SF 2 "vfloat_operand")))]
 503   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)
 504    && VECTOR_UNIT_ALTIVEC_P (V4SImode)"
 505 {
 506   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 507                                     operands[3], operands[4], operands[5]))
 508     DONE;
 509   else
 510     gcc_unreachable ();
 511 })
 512
 513 (define_expand "vconduv2dfv2di"
 514   [(set (match_operand:V2DF 0 "vfloat_operand")
 515         (if_then_else:V2DF
 516          (match_operator 3 "comparison_operator"
 517                          [(match_operand:V2DI 4 "vint_operand")
 518                           (match_operand:V2DI 5 "vint_operand")])
 519          (match_operand:V2DF 1 "vfloat_operand")
 520          (match_operand:V2DF 2 "vfloat_operand")))]
 521   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DFmode)
 522    && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V2DImode)"
 523 {
 524   if (rs6000_emit_vector_cond_expr (operands[0], operands[1], operands[2],
 525                                     operands[3], operands[4], operands[5]))
 526     DONE;
 527   else
 528     gcc_unreachable ();
 529 })
 530
 531 ;; To support vector condition vectorization, define vcond_mask and vec_cmp.
 532
 533 ;; Same mode for condition true/false values and predicate operand.
 534 (define_expand "vcond_mask_<mode><mode>"
 535   [(match_operand:VEC_I 0 "vint_operand")
 536    (match_operand:VEC_I 1 "vint_operand")
 537    (match_operand:VEC_I 2 "vint_operand")
 538    (match_operand:VEC_I 3 "vint_operand")]
 539   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 540 {
 541   emit_insn (gen_vector_select_<mode> (operands[0], operands[2], operands[1],
 542                                   operands[3]));
 543   DONE;
 544 })
 545
 546 ;; Condition true/false values are float but predicate operand is of
 547 ;; type integer vector with same element size.
 548 (define_expand "vcond_mask_<mode><VEC_int>"
 549   [(match_operand:VEC_F 0 "vfloat_operand")
 550    (match_operand:VEC_F 1 "vfloat_operand")
 551    (match_operand:VEC_F 2 "vfloat_operand")
 552    (match_operand:<VEC_INT> 3 "vint_operand")]
 553   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 554 {
 555   emit_insn (gen_vector_select_<mode> (operands[0], operands[2], operands[1],
 556                                   gen_lowpart (<MODE>mode, operands[3])));
 557   DONE;
 558 })
 559
 560 ;; For signed integer vectors comparison.
 561 (define_expand "vec_cmp<mode><mode>"
 562   [(set (match_operand:VEC_IC 0 "vint_operand")
 563         (match_operator 1 "signed_or_equality_comparison_operator"
 564           [(match_operand:VEC_IC 2 "vint_operand")
 565            (match_operand:VEC_IC 3 "vint_operand")]))]
 566   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 567 {
 568   enum rtx_code code = GET_CODE (operands[1]);
 569   rtx tmp = gen_reg_rtx (<MODE>mode);
 570   switch (code)
 571     {
 572     case NE:
 573       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 574       emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[0]));
 575       break;
 576     case EQ:
 577       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 578       break;
 579     case GE:
 580       emit_insn (gen_vector_nlt<mode> (operands[0],operands[2], operands[3],
 581                                        tmp));
 582       break;
 583     case GT:
 584       emit_insn (gen_vector_gt<mode> (operands[0], operands[2], operands[3]));
 585       break;
 586     case LE:
 587       emit_insn (gen_vector_ngt<mode> (operands[0], operands[2], operands[3],
 588                                        tmp));
 589       break;
 590     case LT:
 591       emit_insn (gen_vector_gt<mode> (operands[0], operands[3], operands[2]));
 592       break;
 593     default:
 594       gcc_unreachable ();
 595       break;
 596     }
 597   DONE;
 598 })
 599
 600 ;; For unsigned integer vectors comparison.
 601 (define_expand "vec_cmpu<mode><mode>"
 602   [(set (match_operand:VEC_IC 0 "vint_operand")
 603         (match_operator 1 "unsigned_or_equality_comparison_operator"
 604           [(match_operand:VEC_IC 2 "vint_operand")
 605            (match_operand:VEC_IC 3 "vint_operand")]))]
 606   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 607 {
 608   enum rtx_code code = GET_CODE (operands[1]);
 609   rtx tmp = gen_reg_rtx (<MODE>mode);
 610   switch (code)
 611     {
 612     case NE:
 613       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 614       emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[0]));
 615       break;
 616     case EQ:
 617       emit_insn (gen_vector_eq<mode> (operands[0], operands[2], operands[3]));
 618       break;
 619     case GEU:
 620       emit_insn (gen_vector_nltu<mode> (operands[0], operands[2], operands[3],
 621                                         tmp));
 622       break;
 623     case GTU:
 624       emit_insn (gen_vector_gtu<mode> (operands[0], operands[2], operands[3]));
 625       break;
 626     case LEU:
 627       emit_insn (gen_vector_ngtu<mode> (operands[0], operands[2], operands[3],
 628                                         tmp));
 629       break;
 630     case LTU:
 631       emit_insn (gen_vector_gtu<mode> (operands[0], operands[3], operands[2]));
 632       break;
 633     default:
 634       gcc_unreachable ();
 635       break;
 636     }
 637   DONE;
 638 })
 639
 640 ;; For float point vectors comparison.
 641 (define_expand "vec_cmp<mode><VEC_int>"
 642   [(set (match_operand:<VEC_INT> 0 "vint_operand")
 643          (match_operator 1 "comparison_operator"
 644             [(match_operand:VEC_F 2 "vfloat_operand")
 645             (match_operand:VEC_F 3 "vfloat_operand")]))]
 646   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 647 {
 648   enum rtx_code code = GET_CODE (operands[1]);
 649   rtx res = gen_reg_rtx (<MODE>mode);
 650   switch (code)
 651     {
 652     case NE:
 653       emit_insn (gen_vector_ne<mode> (res, operands[2], operands[3]));
 654       break;
 655     case EQ:
 656       emit_insn (gen_vector_eq<mode> (res, operands[2], operands[3]));
 657       break;
 658     case GE:
 659       emit_insn (gen_vector_ge<mode> (res, operands[2], operands[3]));
 660       break;
 661     case GT:
 662       emit_insn (gen_vector_gt<mode> (res, operands[2], operands[3]));
 663       break;
 664     case LE:
 665       emit_insn (gen_vector_le<mode> (res, operands[2], operands[3]));
 666       break;
 667     case LT:
 668       emit_insn (gen_vector_lt<mode> (res, operands[2], operands[3]));
 669       break;
 670     case LTGT:
 671       emit_insn (gen_vector_ltgt<mode> (res, operands[2], operands[3]));
 672       break;
 673     case UNORDERED:
 674       emit_insn (gen_vector_unordered<mode> (res, operands[2], operands[3]));
 675       break;
 676     case ORDERED:
 677       emit_insn (gen_vector_ordered<mode> (res, operands[2], operands[3]));
 678       break;
 679     case UNEQ:
 680       emit_insn (gen_vector_uneq<mode> (res, operands[2], operands[3]));
 681       break;
 682     case UNGE:
 683       emit_insn (gen_vector_unge<mode> (res, operands[2], operands[3]));
 684       break;
 685     case UNGT:
 686       emit_insn (gen_vector_ungt<mode> (res, operands[2], operands[3]));
 687       break;
 688     case UNLE:
 689       emit_insn (gen_vector_unle<mode> (res, operands[2], operands[3]));
 690       break;
 691     case UNLT:
 692       emit_insn (gen_vector_unlt<mode> (res, operands[2], operands[3]));
 693       break;
 694
 695     default:
 696       gcc_unreachable ();
 697     }
 698
 699   emit_insn (gen_move_insn (operands[0], gen_lowpart (<VEC_INT>mode, res)));
 700   DONE;
 701 })
 702
 703 (define_expand "vector_eq<mode>"
 704   [(set (match_operand:VEC_C 0 "vlogical_operand")
 705         (eq:VEC_C (match_operand:VEC_C 1 "vlogical_operand")
 706                   (match_operand:VEC_C 2 "vlogical_operand")))]
 707   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 708   "")
 709
 710 (define_expand "vector_gt<mode>"
 711   [(set (match_operand:VEC_C 0 "vlogical_operand")
 712         (gt:VEC_C (match_operand:VEC_C 1 "vlogical_operand")
 713                   (match_operand:VEC_C 2 "vlogical_operand")))]
 714   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 715   "")
 716
 717 ; >= for integer vectors: swap operands and apply not-greater-than
 718 (define_expand "vector_nlt<mode>"
 719   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 720         (gt:VEC_IC (match_operand:VEC_IC 2 "vlogical_operand")
 721                    (match_operand:VEC_IC 1 "vlogical_operand")))
 722    (set (match_operand:VEC_IC 0 "vlogical_operand")
 723         (not:VEC_IC (match_dup 3)))]
 724   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 725 {
 726   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 727 })
 728
 729 (define_expand "vector_gtu<mode>"
 730   [(set (match_operand:VEC_IC 0 "vint_operand")
 731         (gtu:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
 732                     (match_operand:VEC_IC 2 "vint_operand")))]
 733   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 734   "")
 735
 736 ; >= for integer vectors: swap operands and apply not-greater-than
 737 (define_expand "vector_nltu<mode>"
 738   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 739         (gtu:VEC_IC (match_operand:VEC_IC 2 "vlogical_operand")
 740                     (match_operand:VEC_IC 1 "vlogical_operand")))
 741    (set (match_operand:VEC_IC 0 "vlogical_operand")
 742         (not:VEC_IC (match_dup 3)))]
 743   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 744 {
 745   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 746 })
 747
 748 (define_expand "vector_geu<mode>"
 749   [(set (match_operand:VEC_IC 0 "vint_operand")
 750         (geu:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
 751                     (match_operand:VEC_IC 2 "vint_operand")))]
 752   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 753   "")
 754
 755 ; <= for integer vectors: apply not-greater-than
 756 (define_expand "vector_ngt<mode>"
 757   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 758         (gt:VEC_IC (match_operand:VEC_IC 1 "vlogical_operand")
 759                    (match_operand:VEC_IC 2 "vlogical_operand")))
 760    (set (match_operand:VEC_IC 0 "vlogical_operand")
 761         (not:VEC_IC (match_dup 3)))]
 762   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 763 {
 764   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 765 })
 766
 767 (define_expand "vector_ngtu<mode>"
 768   [(set (match_operand:VEC_IC 3 "vlogical_operand")
 769         (gtu:VEC_IC (match_operand:VEC_IC 1 "vlogical_operand")
 770                     (match_operand:VEC_IC 2 "vlogical_operand")))
 771    (set (match_operand:VEC_IC 0 "vlogical_operand")
 772         (not:VEC_IC (match_dup 3)))]
 773   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 774 {
 775   operands[3] = gen_reg_rtx_and_attrs (operands[0]);
 776 })
 777
 778 ; There are 14 possible vector FP comparison operators, gt and eq of them have
 779 ; been expanded above, so just support 12 remaining operators here.
 780
 781 ; For ge:
 782 (define_expand "vector_ge<mode>"
 783   [(set (match_operand:VEC_F 0 "vlogical_operand")
 784         (ge:VEC_F (match_operand:VEC_F 1 "vlogical_operand")
 785                   (match_operand:VEC_F 2 "vlogical_operand")))]
 786   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 787   "")
 788
 789 ; For lt/le/ne/ungt/unge/unlt/unle:
 790 ; lt(a,b)   = gt(b,a)
 791 ; le(a,b)   = ge(b,a)
 792 ; unge(a,b) = ~lt(a,b)
 793 ; unle(a,b) = ~gt(a,b)
 794 ; ne(a,b)   = ~eq(a,b)
 795 ; ungt(a,b) = ~le(a,b)
 796 ; unlt(a,b) = ~ge(a,b)
 797 (define_insn_and_split "vector_<code><mode>"
 798   [(set (match_operand:VEC_F 0 "vfloat_operand")
 799         (vector_fp_comparison_simple:VEC_F
 800            (match_operand:VEC_F 1 "vfloat_operand")
 801            (match_operand:VEC_F 2 "vfloat_operand")))]
 802   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode) && can_create_pseudo_p ()"
 803   "#"
 804   "&& can_create_pseudo_p ()"
 805   [(pc)]
 806 {
 807   enum rtx_code cond = <CODE>;
 808   bool need_invert = false;
 809
 810   if (cond == UNLE || cond == UNLT || cond == NE || cond == UNGE
 811       || cond == UNGT)
 812     {
 813       cond = reverse_condition_maybe_unordered (cond);
 814       need_invert = true;
 815     }
 816
 817   if (cond == LT || cond == LE)
 818     {
 819       cond = swap_condition (cond);
 820       std::swap (operands[1], operands[2]);
 821     }
 822
 823   gcc_assert (cond == EQ || cond == GE || cond == GT);
 824
 825   rtx comp = gen_rtx_fmt_ee (cond, <MODE>mode, operands[1], operands[2]);
 826
 827   if (need_invert)
 828     {
 829       rtx res = gen_reg_rtx (<MODE>mode);
 830       emit_insn (gen_rtx_SET (res, comp));
 831       emit_insn (gen_one_cmpl<mode>2 (operands[0], res));
 832     }
 833   else
 834     emit_insn (gen_rtx_SET (operands[0], comp));
 835
 836   DONE;
 837 })
 838
 839 ; For ltgt/uneq/ordered/unordered:
 840 ; ltgt: gt(a,b) | gt(b,a)
 841 ; uneq: ~(gt(a,b) | gt(b,a))
 842 ; ordered: ge(a,b) | ge(b,a)
 843 ; unordered: ~(ge(a,b) | ge(b,a))
 844 (define_insn_and_split "vector_<code><mode>"
 845   [(set (match_operand:VEC_F 0 "vfloat_operand")
 846         (vector_fp_comparison_complex:VEC_F
 847            (match_operand:VEC_F 1 "vfloat_operand")
 848            (match_operand:VEC_F 2 "vfloat_operand")))]
 849   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode) && can_create_pseudo_p ()"
 850   "#"
 851   "&& can_create_pseudo_p ()"
 852   [(pc)]
 853 {
 854   enum rtx_code cond = <CODE>;
 855   bool need_invert = false;
 856
 857   if (cond == UNORDERED || cond == UNEQ)
 858     {
 859       cond = reverse_condition_maybe_unordered (cond);
 860       need_invert = true;
 861     }
 862
 863   if (cond == LTGT)
 864     cond = GT;
 865   else if (cond == ORDERED)
 866     cond = GE;
 867   else
 868     gcc_unreachable ();
 869
 870   rtx comp1 = gen_rtx_fmt_ee (cond, <MODE>mode, operands[1], operands[2]);
 871   rtx res1 = gen_reg_rtx (<MODE>mode);
 872   emit_insn (gen_rtx_SET (res1, comp1));
 873   rtx comp2 = gen_rtx_fmt_ee (cond, <MODE>mode, operands[2], operands[1]);
 874   rtx res2 = gen_reg_rtx (<MODE>mode);
 875   emit_insn (gen_rtx_SET (res2, comp2));
 876
 877   if (need_invert)
 878     {
 879       rtx not1 = gen_rtx_fmt_e (NOT, <MODE>mode, res1);
 880       rtx not2 = gen_rtx_fmt_e (NOT, <MODE>mode, res2);
 881       rtx comp3 = gen_rtx_fmt_ee (AND, <MODE>mode, not1, not2);
 882       emit_insn (gen_rtx_SET (operands[0], comp3));
 883     }
 884   else
 885     emit_insn (gen_ior<mode>3 (operands[0], res1, res2));
 886
 887   DONE;
 888 })
 889
 890 ;; Note the arguments for __builtin_altivec_vsel are op2, op1, mask
 891 ;; which is in the reverse order that we want
 892 (define_expand "vector_select_<mode>"
 893   [(set (match_operand:VEC_L 0 "vlogical_operand")
 894         (ior:VEC_L
 895           (and:VEC_L (not:VEC_L (match_operand:VEC_L 3 "vlogical_operand"))
 896                      (match_operand:VEC_L 1 "vlogical_operand"))
 897           (and:VEC_L (match_dup 3)
 898                      (match_operand:VEC_L 2 "vlogical_operand"))))]
 899   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)")
 900
 901 (define_expand "vector_select_<mode>_uns"
 902   [(set (match_operand:VEC_L 0 "vlogical_operand")
 903         (ior:VEC_L
 904           (and:VEC_L (not:VEC_L (match_operand:VEC_L 3 "vlogical_operand"))
 905                      (match_operand:VEC_L 1 "vlogical_operand"))
 906           (and:VEC_L (match_dup 3)
 907                      (match_operand:VEC_L 2 "vlogical_operand"))))]
 908   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)")
 909
 910 ;; Expansions that compare vectors producing a vector result and a predicate,
 911 ;; setting CR6 to indicate a combined status
 912 (define_expand "vector_eq_<mode>_p"
 913   [(parallel
 914     [(set (reg:CC CR6_REGNO)
 915           (unspec:CC [(eq:CC (match_operand:VEC_A 1 "vlogical_operand")
 916                              (match_operand:VEC_A 2 "vlogical_operand"))]
 917                      UNSPEC_PREDICATE))
 918      (set (match_operand:VEC_A 0 "vlogical_operand")
 919           (eq:VEC_A (match_dup 1)
 920                     (match_dup 2)))])]
 921   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
 922   "")
 923
 924 (define_expand "vector_eq_v1ti_p"
 925   [(parallel
 926     [(set (reg:CC CR6_REGNO)
 927           (unspec:CC [(eq:CC (match_operand:V1TI 1 "altivec_register_operand")
 928                              (match_operand:V1TI 2 "altivec_register_operand"))]
 929                      UNSPEC_PREDICATE))
 930      (set (match_operand:V1TI 0 "vlogical_operand")
 931           (eq:V1TI (match_dup 1)
 932                    (match_dup 2)))])]
 933   "TARGET_POWER10"
 934   "")
 935
 936 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 937 ;; implementation of the vec_all_ne built-in functions on Power9.
 938 (define_expand "vector_ne_<mode>_p"
 939   [(parallel
 940     [(set (reg:CC CR6_REGNO)
 941           (unspec:CC [(ne:CC (match_operand:VI 1 "vlogical_operand")
 942                              (match_operand:VI 2 "vlogical_operand"))]
 943            UNSPEC_PREDICATE))
 944      (set (match_dup 3)
 945           (ne:VI (match_dup 1)
 946                  (match_dup 2)))])
 947    (set (match_operand:SI 0 "register_operand" "=r")
 948         (lt:SI (reg:CC CR6_REGNO)
 949                (const_int 0)))]
 950   "TARGET_P9_VECTOR"
 951 {
 952   operands[3] = gen_reg_rtx (<MODE>mode);
 953 })
 954
 955 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 956 ;; implementation of the vec_any_eq built-in functions on Power9.
 957 (define_expand "vector_ae_<mode>_p"
 958   [(parallel
 959     [(set (reg:CC CR6_REGNO)
 960           (unspec:CC [(ne:CC (match_operand:VI 1 "vlogical_operand")
 961                              (match_operand:VI 2 "vlogical_operand"))]
 962            UNSPEC_PREDICATE))
 963      (set (match_dup 3)
 964           (ne:VI (match_dup 1)
 965                  (match_dup 2)))])
 966    (set (match_operand:SI 0 "register_operand" "=r")
 967         (lt:SI (reg:CC CR6_REGNO)
 968                (const_int 0)))
 969    (set (match_dup 0)
 970         (xor:SI (match_dup 0)
 971                 (const_int 1)))]
 972   "TARGET_P9_VECTOR"
 973 {
 974   operands[3] = gen_reg_rtx (<MODE>mode);
 975 })
 976
 977 ;; This expansion handles the V16QI, V8HI, and V4SI modes in the
 978 ;; implementation of the vec_all_nez and vec_any_eqz built-in
 979 ;; functions on Power9.
 980 (define_expand "vector_nez_<mode>_p"
 981   [(parallel
 982     [(set (reg:CC CR6_REGNO)
 983           (unspec:CC [(unspec:VI
 984                        [(match_operand:VI 1 "vlogical_operand")
 985                         (match_operand:VI 2 "vlogical_operand")]
 986                        UNSPEC_NEZ_P)]
 987            UNSPEC_PREDICATE))
 988      (set (match_operand:VI 0 "vlogical_operand")
 989           (unspec:VI [(match_dup 1)
 990                       (match_dup 2)]
 991            UNSPEC_NEZ_P))])]
 992   "TARGET_P9_VECTOR"
 993   "")
 994
 995 ;; This expansion handles the V2DI mode in the implementation of the
 996 ;; vec_all_ne built-in function on Power9.
 997 ;;
 998 ;; Since the Power9 "xvcmpne<mode>." instruction does not support DImode,
 999 ;; this expands into the same rtl that would be used for the Power8
1000 ;; architecture.
1001 (define_expand "vector_ne_v2di_p"
1002   [(parallel
1003     [(set (reg:CC CR6_REGNO)
1004           (unspec:CC [(eq:CC (match_operand:V2DI 1 "vlogical_operand")
1005                              (match_operand:V2DI 2 "vlogical_operand"))]
1006                      UNSPEC_PREDICATE))
1007      (set (match_dup 3)
1008           (eq:V2DI (match_dup 1)
1009                    (match_dup 2)))])
1010    (set (match_operand:SI 0 "register_operand" "=r")
1011         (eq:SI (reg:CC CR6_REGNO)
1012                (const_int 0)))]
1013   "TARGET_P9_VECTOR"
1014 {
1015   operands[3] = gen_reg_rtx (V2DImode);
1016 })
1017
1018 (define_expand "vector_ne_v1ti_p"
1019   [(parallel
1020     [(set (reg:CC CR6_REGNO)
1021           (unspec:CC [(eq:CC (match_operand:V1TI 1 "altivec_register_operand")
1022                              (match_operand:V1TI 2 "altivec_register_operand"))]
1023                      UNSPEC_PREDICATE))
1024      (set (match_dup 3)
1025           (eq:V1TI (match_dup 1)
1026                    (match_dup 2)))])
1027    (set (match_operand:SI 0 "register_operand" "=r")
1028         (eq:SI (reg:CC CR6_REGNO)
1029                (const_int 0)))]
1030   "TARGET_POWER10"
1031 {
1032   operands[3] = gen_reg_rtx (V1TImode);
1033 })
1034
1035 ;; This expansion handles the V2DI mode in the implementation of the
1036 ;; vec_any_eq built-in function on Power9.
1037 ;;
1038 ;; Since the Power9 "xvcmpne<mode>." instruction does not support DImode,
1039 ;; this expands into the same rtl that would be used for the Power8
1040 ;; architecture.
1041 (define_expand "vector_ae_v2di_p"
1042   [(parallel
1043     [(set (reg:CC CR6_REGNO)
1044           (unspec:CC [(eq:CC (match_operand:V2DI 1 "vlogical_operand")
1045                              (match_operand:V2DI 2 "vlogical_operand"))]
1046                      UNSPEC_PREDICATE))
1047      (set (match_dup 3)
1048           (eq:V2DI (match_dup 1)
1049                    (match_dup 2)))])
1050    (set (match_operand:SI 0 "register_operand" "=r")
1051         (eq:SI (reg:CC CR6_REGNO)
1052                (const_int 0)))
1053    (set (match_dup 0)
1054         (xor:SI (match_dup 0)
1055                 (const_int 1)))]
1056   "TARGET_P9_VECTOR"
1057 {
1058   operands[3] = gen_reg_rtx (V2DImode);
1059 })
1060
1061 (define_expand "vector_ae_v1ti_p"
1062   [(parallel
1063     [(set (reg:CC CR6_REGNO)
1064           (unspec:CC [(eq:CC (match_operand:V1TI 1 "altivec_register_operand")
1065                              (match_operand:V1TI 2 "altivec_register_operand"))]
1066                      UNSPEC_PREDICATE))
1067      (set (match_dup 3)
1068           (eq:V1TI (match_dup 1)
1069                    (match_dup 2)))])
1070    (set (match_operand:SI 0 "register_operand" "=r")
1071         (eq:SI (reg:CC CR6_REGNO)
1072                (const_int 0)))
1073    (set (match_dup 0)
1074         (xor:SI (match_dup 0)
1075                 (const_int 1)))]
1076   "TARGET_POWER10"
1077 {
1078   operands[3] = gen_reg_rtx (V1TImode);
1079 })
1080
1081 ;; This expansion handles the V4SF and V2DF modes in the Power9
1082 ;; implementation of the vec_all_ne built-in functions.  Note that the
1083 ;; expansions for this pattern with these modes makes no use of power9-
1084 ;; specific instructions since there are no new power9 instructions
1085 ;; for vector compare not equal with floating point arguments.
1086 (define_expand "vector_ne_<mode>_p"
1087   [(parallel
1088     [(set (reg:CC CR6_REGNO)
1089           (unspec:CC [(eq:CC (match_operand:VEC_F 1 "vlogical_operand")
1090                              (match_operand:VEC_F 2 "vlogical_operand"))]
1091                      UNSPEC_PREDICATE))
1092      (set (match_dup 3)
1093           (eq:VEC_F (match_dup 1)
1094                     (match_dup 2)))])
1095    (set (match_operand:SI 0 "register_operand" "=r")
1096         (eq:SI (reg:CC CR6_REGNO)
1097                (const_int 0)))]
1098   "TARGET_P9_VECTOR"
1099 {
1100   operands[3] = gen_reg_rtx (<MODE>mode);
1101 })
1102
1103 ;; This expansion handles the V4SF and V2DF modes in the Power9
1104 ;; implementation of the vec_any_eq built-in functions.  Note that the
1105 ;; expansions for this pattern with these modes makes no use of power9-
1106 ;; specific instructions since there are no new power9 instructions
1107 ;; for vector compare not equal with floating point arguments.
1108 (define_expand "vector_ae_<mode>_p"
1109   [(parallel
1110     [(set (reg:CC CR6_REGNO)
1111           (unspec:CC [(eq:CC (match_operand:VEC_F 1 "vlogical_operand")
1112                              (match_operand:VEC_F 2 "vlogical_operand"))]
1113                      UNSPEC_PREDICATE))
1114      (set (match_dup 3)
1115           (eq:VEC_F (match_dup 1)
1116                     (match_dup 2)))])
1117    (set (match_operand:SI 0 "register_operand" "=r")
1118         (eq:SI (reg:CC CR6_REGNO)
1119                (const_int 0)))
1120    (set (match_dup 0)
1121         (xor:SI (match_dup 0)
1122                 (const_int 1)))]
1123   "TARGET_P9_VECTOR"
1124 {
1125   operands[3] = gen_reg_rtx (<MODE>mode);
1126 })
1127
1128 (define_expand "vector_gt_<mode>_p"
1129   [(parallel
1130     [(set (reg:CC CR6_REGNO)
1131           (unspec:CC [(gt:CC (match_operand:VEC_A 1 "vlogical_operand")
1132                              (match_operand:VEC_A 2 "vlogical_operand"))]
1133                      UNSPEC_PREDICATE))
1134      (set (match_operand:VEC_A 0 "vlogical_operand")
1135           (gt:VEC_A (match_dup 1)
1136                     (match_dup 2)))])]
1137   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1138   "")
1139
1140 (define_expand "vector_gt_v1ti_p"
1141   [(parallel
1142     [(set (reg:CC CR6_REGNO)
1143           (unspec:CC [(gt:CC (match_operand:V1TI 1 "vlogical_operand")
1144                              (match_operand:V1TI 2 "vlogical_operand"))]
1145                      UNSPEC_PREDICATE))
1146      (set (match_operand:V1TI 0 "vlogical_operand")
1147           (gt:V1TI (match_dup 1)
1148                    (match_dup 2)))])]
1149   "TARGET_POWER10"
1150   "")
1151
1152 (define_expand "vector_ge_<mode>_p"
1153   [(parallel
1154     [(set (reg:CC CR6_REGNO)
1155           (unspec:CC [(ge:CC (match_operand:VEC_F 1 "vfloat_operand")
1156                              (match_operand:VEC_F 2 "vfloat_operand"))]
1157                      UNSPEC_PREDICATE))
1158      (set (match_operand:VEC_F 0 "vfloat_operand")
1159           (ge:VEC_F (match_dup 1)
1160                     (match_dup 2)))])]
1161   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1162   "")
1163
1164 (define_expand "vector_gtu_<mode>_p"
1165   [(parallel
1166     [(set (reg:CC CR6_REGNO)
1167           (unspec:CC [(gtu:CC (match_operand:VEC_IC 1 "vint_operand")
1168                               (match_operand:VEC_IC 2 "vint_operand"))]
1169                      UNSPEC_PREDICATE))
1170      (set (match_operand:VEC_IC 0 "vlogical_operand")
1171           (gtu:VEC_IC (match_dup 1)
1172                       (match_dup 2)))])]
1173   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1174   "")
1175
1176 ;; AltiVec/VSX predicates.
1177
1178 ;; This expansion is triggered during expansion of predicate built-in
1179 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1180 ;; altivec_expand_predicate_builtin() function when the value of the
1181 ;; integer constant first argument equals zero (aka __CR6_EQ in altivec.h).
1182 (define_expand "cr6_test_for_zero"
1183   [(set (match_operand:SI 0 "register_operand" "=r")
1184         (eq:SI (reg:CC CR6_REGNO)
1185                (const_int 0)))]
1186   "TARGET_ALTIVEC || TARGET_VSX"
1187   "")
1188
1189 ;; This expansion is triggered during expansion of predicate built-in
1190 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1191 ;; altivec_expand_predicate_builtin() function when the value of the
1192 ;; integer constant first argument equals one (aka __CR6_EQ_REV in altivec.h).
1193 (define_expand "cr6_test_for_zero_reverse"
1194   [(set (match_operand:SI 0 "register_operand" "=r")
1195         (eq:SI (reg:CC CR6_REGNO)
1196                (const_int 0)))
1197    (set (match_dup 0)
1198         (xor:SI (match_dup 0)
1199                 (const_int 1)))]
1200   "TARGET_ALTIVEC || TARGET_VSX"
1201   "")
1202
1203 ;; This expansion is triggered during expansion of predicate built-in
1204 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1205 ;; altivec_expand_predicate_builtin() function when the value of the
1206 ;; integer constant first argument equals two (aka __CR6_LT in altivec.h).
1207 (define_expand "cr6_test_for_lt"
1208   [(set (match_operand:SI 0 "register_operand" "=r")
1209         (lt:SI (reg:CC CR6_REGNO)
1210                (const_int 0)))]
1211   "TARGET_ALTIVEC || TARGET_VSX"
1212   "")
1213
1214 ;; This expansion is triggered during expansion of predicate built-in
1215 ;; functions (built-ins defined with the RS6000_BUILTIN_P macro) by the
1216 ;; altivec_expand_predicate_builtin() function when the value of the
1217 ;; integer constant first argument equals three
1218 ;; (aka __CR6_LT_REV in altivec.h).
1219 (define_expand "cr6_test_for_lt_reverse"
1220   [(set (match_operand:SI 0 "register_operand" "=r")
1221         (lt:SI (reg:CC CR6_REGNO)
1222                (const_int 0)))
1223    (set (match_dup 0)
1224         (xor:SI (match_dup 0)
1225                 (const_int 1)))]
1226   "TARGET_ALTIVEC || TARGET_VSX"
1227   "")
1228
1229 \f
1230 ;; Vector count leading zeros
1231 (define_expand "clz<mode>2"
1232   [(set (match_operand:VEC_I 0 "register_operand")
1233         (clz:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1234   "TARGET_P8_VECTOR")
1235
1236 ;; Vector count trailing zeros
1237 (define_expand "ctz<mode>2"
1238   [(set (match_operand:VEC_I 0 "register_operand")
1239         (ctz:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1240   "TARGET_P9_VECTOR")
1241
1242 ;; Vector population count
1243 (define_expand "popcount<mode>2"
1244   [(set (match_operand:VEC_I 0 "register_operand")
1245         (popcount:VEC_I (match_operand:VEC_I 1 "register_operand")))]
1246   "TARGET_P8_VECTOR")
1247
1248 ;; Vector parity
1249 (define_expand "parity<mode>2"
1250   [(set (match_operand:VEC_IP 0 "register_operand")
1251         (parity:VEC_IP (match_operand:VEC_IP 1 "register_operand")))]
1252   "TARGET_P9_VECTOR"
1253 {
1254   rtx op1 = gen_lowpart (V16QImode, operands[1]);
1255   rtx res = gen_reg_rtx (V16QImode);
1256   emit_insn (gen_popcountv16qi2 (res, op1));
1257   emit_insn (gen_rs6000_vprtyb<mode>2 (operands[0],
1258                                        gen_lowpart (<MODE>mode, res)));
1259
1260   DONE;
1261 })
1262
1263 \f
1264 ;; Same size conversions
1265 (define_expand "float<VEC_int><mode>2"
1266   [(set (match_operand:VEC_F 0 "vfloat_operand")
1267         (float:VEC_F (match_operand:<VEC_INT> 1 "vint_operand")))]
1268   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1269 {
1270   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1271     {
1272       emit_insn (gen_altivec_vcfsx (operands[0], operands[1], const0_rtx));
1273       DONE;
1274     }
1275 })
1276
1277 (define_expand "floatuns<VEC_int><mode>2"
1278   [(set (match_operand:VEC_F 0 "vfloat_operand")
1279         (unsigned_float:VEC_F (match_operand:<VEC_INT> 1 "vint_operand")))]
1280   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1281 {
1282   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1283     {
1284       emit_insn (gen_altivec_vcfux (operands[0], operands[1], const0_rtx));
1285       DONE;
1286     }
1287 })
1288
1289 (define_expand "fix_trunc<mode><VEC_int>2"
1290   [(set (match_operand:<VEC_INT> 0 "vint_operand")
1291         (fix:<VEC_INT> (match_operand:VEC_F 1 "vfloat_operand")))]
1292   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1293 {
1294   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1295     {
1296       emit_insn (gen_altivec_vctsxs (operands[0], operands[1], const0_rtx));
1297       DONE;
1298     }
1299 })
1300
1301 (define_expand "fixuns_trunc<mode><VEC_int>2"
1302   [(set (match_operand:<VEC_INT> 0 "vint_operand")
1303         (unsigned_fix:<VEC_INT> (match_operand:VEC_F 1 "vfloat_operand")))]
1304   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1305 {
1306   if (<MODE>mode == V4SFmode && VECTOR_UNIT_ALTIVEC_P (<MODE>mode))
1307     {
1308       emit_insn (gen_altivec_vctuxs (operands[0], operands[1], const0_rtx));
1309       DONE;
1310     }
1311 })
1312
1313 \f
1314 ;; Vector initialization, set, extract
1315 (define_expand "vec_init<mode><VEC_base_l>"
1316   [(match_operand:VEC_E 0 "vlogical_operand")
1317    (match_operand:VEC_E 1 "")]
1318   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1319 {
1320   rs6000_expand_vector_init (operands[0], operands[1]);
1321   DONE;
1322 })
1323
1324 (define_expand "vec_set<mode>"
1325   [(match_operand:VEC_E 0 "vlogical_operand")
1326    (match_operand:<VEC_base> 1 "register_operand")
1327    (match_operand 2 "vec_set_index_operand")]
1328   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1329 {
1330   rs6000_expand_vector_set (operands[0], operands[1], operands[2]);
1331   DONE;
1332 })
1333
1334 (define_expand "vec_extract<mode><VEC_base_l>"
1335   [(match_operand:<VEC_base> 0 "register_operand")
1336    (match_operand:VEC_E 1 "vlogical_operand")
1337    (match_operand 2 "const_int_operand")]
1338   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1339 {
1340   rs6000_expand_vector_extract (operands[0], operands[1], operands[2]);
1341   DONE;
1342 })
1343 \f
1344 ;; Convert double word types to single word types
1345 (define_expand "vec_pack_trunc_v2df"
1346   [(match_operand:V4SF 0 "vfloat_operand")
1347    (match_operand:V2DF 1 "vfloat_operand")
1348    (match_operand:V2DF 2 "vfloat_operand")]
1349   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1350 {
1351   rtx r1 = gen_reg_rtx (V4SFmode);
1352   rtx r2 = gen_reg_rtx (V4SFmode);
1353
1354   emit_insn (gen_vsx_xvcvdpsp (r1, operands[1]));
1355   emit_insn (gen_vsx_xvcvdpsp (r2, operands[2]));
1356   rs6000_expand_extract_even (operands[0], r1, r2);
1357   DONE;
1358 })
1359
1360 (define_expand "vec_pack_sfix_trunc_v2df"
1361   [(match_operand:V4SI 0 "vint_operand")
1362    (match_operand:V2DF 1 "vfloat_operand")
1363    (match_operand:V2DF 2 "vfloat_operand")]
1364   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1365 {
1366   rtx r1 = gen_reg_rtx (V4SImode);
1367   rtx r2 = gen_reg_rtx (V4SImode);
1368
1369   emit_insn (gen_vsx_xvcvdpsxws (r1, operands[1]));
1370   emit_insn (gen_vsx_xvcvdpsxws (r2, operands[2]));
1371   rs6000_expand_extract_even (operands[0], r1, r2);
1372   DONE;
1373 })
1374
1375 (define_expand "vec_pack_ufix_trunc_v2df"
1376   [(match_operand:V4SI 0 "vint_operand")
1377    (match_operand:V2DF 1 "vfloat_operand")
1378    (match_operand:V2DF 2 "vfloat_operand")]
1379   "VECTOR_UNIT_VSX_P (V2DFmode) && TARGET_ALTIVEC"
1380 {
1381   rtx r1 = gen_reg_rtx (V4SImode);
1382   rtx r2 = gen_reg_rtx (V4SImode);
1383
1384   emit_insn (gen_vsx_xvcvdpuxws (r1, operands[1]));
1385   emit_insn (gen_vsx_xvcvdpuxws (r2, operands[2]));
1386   rs6000_expand_extract_even (operands[0], r1, r2);
1387   DONE;
1388 })
1389
1390 ;; Convert single word types to double word
1391 (define_expand "vec_unpacks_hi_v4sf"
1392   [(match_operand:V2DF 0 "vfloat_operand")
1393    (match_operand:V4SF 1 "vfloat_operand")]
1394   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)"
1395 {
1396   rtx reg = gen_reg_rtx (V4SFmode);
1397
1398   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1399   emit_insn (gen_vsx_xvcvspdp (operands[0], reg));
1400   DONE;
1401 })
1402
1403 (define_expand "vec_unpacks_lo_v4sf"
1404   [(match_operand:V2DF 0 "vfloat_operand")
1405    (match_operand:V4SF 1 "vfloat_operand")]
1406   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SFmode)"
1407 {
1408   rtx reg = gen_reg_rtx (V4SFmode);
1409
1410   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1411   emit_insn (gen_vsx_xvcvspdp (operands[0], reg));
1412   DONE;
1413 })
1414
1415 (define_expand "vec_unpacks_float_hi_v4si"
1416   [(match_operand:V2DF 0 "vfloat_operand")
1417    (match_operand:V4SI 1 "vint_operand")]
1418   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1419 {
1420   rtx reg = gen_reg_rtx (V4SImode);
1421
1422   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1423   emit_insn (gen_vsx_xvcvsxwdp (operands[0], reg));
1424   DONE;
1425 })
1426
1427 (define_expand "vec_unpacks_float_lo_v4si"
1428   [(match_operand:V2DF 0 "vfloat_operand")
1429    (match_operand:V4SI 1 "vint_operand")]
1430   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1431 {
1432   rtx reg = gen_reg_rtx (V4SImode);
1433
1434   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1435   emit_insn (gen_vsx_xvcvsxwdp (operands[0], reg));
1436   DONE;
1437 })
1438
1439 (define_expand "vec_unpacku_float_hi_v4si"
1440   [(match_operand:V2DF 0 "vfloat_operand")
1441    (match_operand:V4SI 1 "vint_operand")]
1442   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1443 {
1444   rtx reg = gen_reg_rtx (V4SImode);
1445
1446   rs6000_expand_interleave (reg, operands[1], operands[1], BYTES_BIG_ENDIAN);
1447   emit_insn (gen_vsx_xvcvuxwdp (operands[0], reg));
1448   DONE;
1449 })
1450
1451 (define_expand "vec_unpacku_float_lo_v4si"
1452   [(match_operand:V2DF 0 "vfloat_operand")
1453    (match_operand:V4SI 1 "vint_operand")]
1454   "VECTOR_UNIT_VSX_P (V2DFmode) && VECTOR_UNIT_ALTIVEC_OR_VSX_P (V4SImode)"
1455 {
1456   rtx reg = gen_reg_rtx (V4SImode);
1457
1458   rs6000_expand_interleave (reg, operands[1], operands[1], !BYTES_BIG_ENDIAN);
1459   emit_insn (gen_vsx_xvcvuxwdp (operands[0], reg));
1460   DONE;
1461 })
1462
1463 \f
1464 ;; Align vector loads with a permute.
1465 (define_expand "vec_realign_load_<mode>"
1466   [(match_operand:VEC_K 0 "vlogical_operand")
1467    (match_operand:VEC_K 1 "vlogical_operand")
1468    (match_operand:VEC_K 2 "vlogical_operand")
1469    (match_operand:V16QI 3 "vlogical_operand")]
1470   "VECTOR_MEM_ALTIVEC_OR_VSX_P (<MODE>mode)"
1471 {
1472   if (BYTES_BIG_ENDIAN)
1473     emit_insn (gen_altivec_vperm_<mode> (operands[0], operands[1],
1474                                          operands[2], operands[3]));
1475   else
1476     {
1477       /* We have changed lvsr to lvsl, so to complete the transformation
1478          of vperm for LE, we must swap the inputs.  */
1479       rtx unspec = gen_rtx_UNSPEC (<MODE>mode,
1480                                    gen_rtvec (3, operands[2],
1481                                               operands[1], operands[3]),
1482                                    UNSPEC_VPERM);
1483       emit_move_insn (operands[0], unspec);
1484     }
1485   DONE;
1486 })
1487
1488 ;; Under VSX, vectors of 4/8 byte alignments do not need to be aligned
1489 ;; since the load already handles it.
1490 (define_expand "movmisalign<mode>"
1491  [(set (match_operand:VEC_N 0 "nonimmediate_operand")
1492        (match_operand:VEC_N 1 "any_operand"))]
1493  "VECTOR_MEM_VSX_P (<MODE>mode) && TARGET_ALLOW_MOVMISALIGN"
1494 {
1495   rs6000_emit_move (operands[0], operands[1], <MODE>mode);
1496   DONE;
1497 })
1498
1499 ;; Vector shift right in bits. Currently supported ony for shift
1500 ;; amounts that can be expressed as byte shifts (divisible by 8).
1501 ;; General shift amounts can be supported using vsro + vsr. We're
1502 ;; not expecting to see these yet (the vectorizer currently
1503 ;; generates only shifts by a whole number of vector elements).
1504 ;; Note that the vec_shr operation is actually defined as
1505 ;; 'shift toward element 0' so is a shr for LE and shl for BE.
1506 (define_expand "vec_shr_<mode>"
1507   [(match_operand:VEC_L 0 "vlogical_operand")
1508    (match_operand:VEC_L 1 "vlogical_operand")
1509    (match_operand:QI 2 "reg_or_short_operand")]
1510   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1511 {
1512   rtx bitshift = operands[2];
1513   rtx shift;
1514   rtx insn;
1515   rtx zero_reg, op1, op2;
1516   HOST_WIDE_INT bitshift_val;
1517   HOST_WIDE_INT byteshift_val;
1518
1519   if (! CONSTANT_P (bitshift))
1520     FAIL;
1521   bitshift_val = INTVAL (bitshift);
1522   if (bitshift_val & 0x7)
1523     FAIL;
1524   byteshift_val = (bitshift_val >> 3);
1525   zero_reg = gen_reg_rtx (<MODE>mode);
1526   emit_move_insn (zero_reg, CONST0_RTX (<MODE>mode));
1527   if (!BYTES_BIG_ENDIAN)
1528     {
1529       /* Note, byteshift_val can be 0!  */
1530       byteshift_val = -byteshift_val & 15;
1531       op1 = zero_reg;
1532       op2 = operands[1];
1533     }
1534   else
1535     {
1536       op1 = operands[1];
1537       op2 = zero_reg;
1538     }
1539
1540   if (TARGET_VSX && (byteshift_val & 0x3) == 0)
1541     {
1542       shift = gen_rtx_CONST_INT (QImode, byteshift_val >> 2);
1543       insn = gen_vsx_xxsldwi_<mode> (operands[0], op1, op2, shift);
1544     }
1545   else
1546     {
1547       shift = gen_rtx_CONST_INT (QImode, byteshift_val);
1548       insn = gen_altivec_vsldoi_<mode> (operands[0], op1, op2, shift);
1549     }
1550
1551   emit_insn (insn);
1552   DONE;
1553 })
1554
1555 ;; Expanders for rotate each element in a vector
1556 (define_expand "vrotl<mode>3"
1557   [(set (match_operand:VEC_IC 0 "vint_operand")
1558         (rotate:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
1559                        (match_operand:VEC_IC 2 "vint_operand")))]
1560   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1561 {
1562   /* Shift amount in needs to be put in bits[57:63] of 128-bit operand2.  */
1563   if (<MODE>mode == V1TImode)
1564     {
1565       rtx tmp = gen_reg_rtx (V1TImode);
1566
1567       emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1568       emit_insn (gen_altivec_vrlq (operands[0], operands[1], tmp));
1569       DONE;
1570     }
1571  })
1572
1573 ;; Expanders for rotatert to make use of vrotl
1574 (define_expand "vrotr<mode>3"
1575   [(set (match_operand:VEC_I 0 "vint_operand")
1576         (rotatert:VEC_I (match_operand:VEC_I 1 "vint_operand")
1577                 (match_operand:VEC_I 2 "vint_operand")))]
1578   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1579 {
1580   rtx rot_count = gen_reg_rtx (<MODE>mode);
1581   emit_insn (gen_neg<mode>2 (rot_count, operands[2]));
1582   emit_insn (gen_vrotl<mode>3 (operands[0], operands[1], rot_count));
1583   DONE;
1584 })
1585
1586 ;; Expanders for arithmetic shift left on each vector element
1587 (define_expand "vashl<mode>3"
1588   [(set (match_operand:VEC_I 0 "vint_operand")
1589         (ashift:VEC_I (match_operand:VEC_I 1 "vint_operand")
1590                       (match_operand:VEC_I 2 "vint_operand")))]
1591   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1592   "")
1593
1594 ;; No immediate version of this 128-bit instruction
1595 (define_expand "vashl<mode>3"
1596   [(set (match_operand:VEC_TI 0 "vsx_register_operand" "=v")
1597         (ashift:VEC_TI (match_operand:VEC_TI 1 "vsx_register_operand")
1598                          (match_operand:VEC_TI 2 "vsx_register_operand")))]
1599   "TARGET_POWER10"
1600 {
1601   /* Shift amount in needs to be put in bits[57:63] of 128-bit operand2. */
1602   rtx tmp = gen_reg_rtx (<MODE>mode);
1603
1604   emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1605   emit_insn(gen_altivec_vslq_<mode> (operands[0], operands[1], tmp));
1606   DONE;
1607 })
1608
1609 ;; Expanders for logical shift right on each vector element
1610 (define_expand "vlshr<mode>3"
1611   [(set (match_operand:VEC_I 0 "vint_operand")
1612         (lshiftrt:VEC_I (match_operand:VEC_I 1 "vint_operand")
1613                         (match_operand:VEC_I 2 "vint_operand")))]
1614   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1615   "")
1616
1617 ;; No immediate version of this 128-bit instruction
1618 (define_expand "vlshr<mode>3"
1619   [(set (match_operand:VEC_TI 0 "vsx_register_operand" "=v")
1620         (lshiftrt:VEC_TI (match_operand:VEC_TI 1 "vsx_register_operand")
1621                            (match_operand:VEC_TI 2 "vsx_register_operand")))]
1622   "TARGET_POWER10"
1623 {
1624   /* Shift amount in needs to be put into bits[57:63] of 128-bit operand2. */
1625   rtx tmp = gen_reg_rtx (<MODE>mode);
1626
1627   emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1628   emit_insn(gen_altivec_vsrq_<mode> (operands[0], operands[1], tmp));
1629   DONE;
1630 })
1631
1632 ;; Expanders for arithmetic shift right on each vector element
1633 (define_expand "vashr<mode>3"
1634   [(set (match_operand:VEC_IC 0 "vint_operand")
1635         (ashiftrt:VEC_IC (match_operand:VEC_IC 1 "vint_operand")
1636                          (match_operand:VEC_IC 2 "vint_operand")))]
1637   "VECTOR_UNIT_ALTIVEC_OR_VSX_P (<MODE>mode)"
1638 {
1639   /* Shift amount in needs to be put in bits[57:63] of 128-bit operand2.  */
1640   if (<MODE>mode == V1TImode)
1641     {
1642       rtx tmp = gen_reg_rtx (V1TImode);
1643
1644       emit_insn (gen_xxswapd_v1ti (tmp, operands[2]));
1645       emit_insn (gen_altivec_vsraq (operands[0], operands[1], tmp));
1646       DONE;
1647     }
1648 })
1649
1650 \f
1651 ;; Vector reduction expanders for VSX
1652 ; The (VEC_reduc:...
1653 ;       (op1)
1654 ;       (unspec:... [(const_int 0)] UNSPEC_REDUC))
1655 ;
1656 ; is to allow us to use a code iterator, but not completely list all of the
1657 ; vector rotates, etc. to prevent canonicalization
1658
1659
1660 (define_expand "reduc_<VEC_reduc:VEC_reduc_name>_scal_<VEC_F:mode>"
1661   [(match_operand:<VEC_base> 0 "register_operand")
1662    (VEC_reduc:VEC_F (match_operand:VEC_F 1 "vfloat_operand")
1663                     (unspec:VEC_F [(const_int 0)] UNSPEC_REDUC))]
1664   "VECTOR_UNIT_VSX_P (<VEC_F:MODE>mode)"
1665   {
1666     rtx vec = gen_reg_rtx (<VEC_F:MODE>mode);
1667     rtx elt = BYTES_BIG_ENDIAN
1668                 ? gen_int_mode (GET_MODE_NUNITS (<VEC_F:MODE>mode) - 1, QImode)
1669                 : const0_rtx;
1670     emit_insn (gen_vsx_reduc_<VEC_reduc:VEC_reduc_name>_<VEC_F:mode> (vec,
1671         operand1));
1672     emit_insn (gen_vsx_extract_<VEC_F:mode> (operand0, vec, elt));
1673     DONE;
1674   })