gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2018 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V4_REGNUM          36)
  61     (V8_REGNUM          40)
  62     (V12_REGNUM         44)
  63     (V15_REGNUM         47)
  64     (V16_REGNUM         48)
  65     (V20_REGNUM         52)
  66     (V24_REGNUM         56)
  67     (V28_REGNUM         60)
  68     (V31_REGNUM         63)
  69     (LAST_SAVED_REGNUM  63)
  70     (SFP_REGNUM         64)
  71     (AP_REGNUM          65)
  72     (CC_REGNUM          66)
  73     ;; Defined only to make the DWARF description simpler.
  74     (VG_REGNUM          67)
  75     (P0_REGNUM          68)
  76     (P1_REGNUM          69)
  77     (P2_REGNUM          70)
  78     (P3_REGNUM          71)
  79     (P4_REGNUM          72)
  80     (P5_REGNUM          73)
  81     (P6_REGNUM          74)
  82     (P7_REGNUM          75)
  83     (P8_REGNUM          76)
  84     (P9_REGNUM          77)
  85     (P10_REGNUM         78)
  86     (P11_REGNUM         79)
  87     (P12_REGNUM         80)
  88     (P13_REGNUM         81)
  89     (P14_REGNUM         82)
  90     (P15_REGNUM         83)
  91   ]
  92 )
  93
  94 (define_c_enum "unspec" [
  95     UNSPEC_AUTI1716
  96     UNSPEC_AUTISP
  97     UNSPEC_CASESI
  98     UNSPEC_CRC32B
  99     UNSPEC_CRC32CB
 100     UNSPEC_CRC32CH
 101     UNSPEC_CRC32CW
 102     UNSPEC_CRC32CX
 103     UNSPEC_CRC32H
 104     UNSPEC_CRC32W
 105     UNSPEC_CRC32X
 106     UNSPEC_FCVTZS
 107     UNSPEC_FCVTZU
 108     UNSPEC_URECPE
 109     UNSPEC_FRECPE
 110     UNSPEC_FRECPS
 111     UNSPEC_FRECPX
 112     UNSPEC_FRINTA
 113     UNSPEC_FRINTI
 114     UNSPEC_FRINTM
 115     UNSPEC_FRINTN
 116     UNSPEC_FRINTP
 117     UNSPEC_FRINTX
 118     UNSPEC_FRINTZ
 119     UNSPEC_GOTSMALLPIC
 120     UNSPEC_GOTSMALLPIC28K
 121     UNSPEC_GOTSMALLTLS
 122     UNSPEC_GOTTINYPIC
 123     UNSPEC_GOTTINYTLS
 124     UNSPEC_LD1
 125     UNSPEC_LD2
 126     UNSPEC_LD2_DREG
 127     UNSPEC_LD2_DUP
 128     UNSPEC_LD3
 129     UNSPEC_LD3_DREG
 130     UNSPEC_LD3_DUP
 131     UNSPEC_LD4
 132     UNSPEC_LD4_DREG
 133     UNSPEC_LD4_DUP
 134     UNSPEC_LD2_LANE
 135     UNSPEC_LD3_LANE
 136     UNSPEC_LD4_LANE
 137     UNSPEC_MB
 138     UNSPEC_NOP
 139     UNSPEC_PACI1716
 140     UNSPEC_PACISP
 141     UNSPEC_PRLG_STK
 142     UNSPEC_REV
 143     UNSPEC_RBIT
 144     UNSPEC_SABAL
 145     UNSPEC_SABDL2
 146     UNSPEC_SADALP
 147     UNSPEC_SCVTF
 148     UNSPEC_SISD_NEG
 149     UNSPEC_SISD_SSHL
 150     UNSPEC_SISD_USHL
 151     UNSPEC_SSHL_2S
 152     UNSPEC_ST1
 153     UNSPEC_ST2
 154     UNSPEC_ST3
 155     UNSPEC_ST4
 156     UNSPEC_ST2_LANE
 157     UNSPEC_ST3_LANE
 158     UNSPEC_ST4_LANE
 159     UNSPEC_TLS
 160     UNSPEC_TLSDESC
 161     UNSPEC_TLSLE12
 162     UNSPEC_TLSLE24
 163     UNSPEC_TLSLE32
 164     UNSPEC_TLSLE48
 165     UNSPEC_UABAL
 166     UNSPEC_UABDL2
 167     UNSPEC_UADALP
 168     UNSPEC_UCVTF
 169     UNSPEC_USHL_2S
 170     UNSPEC_VSTRUCTDUMMY
 171     UNSPEC_SP_SET
 172     UNSPEC_SP_TEST
 173     UNSPEC_RSQRT
 174     UNSPEC_RSQRTE
 175     UNSPEC_RSQRTS
 176     UNSPEC_NZCV
 177     UNSPEC_XPACLRI
 178     UNSPEC_LD1_SVE
 179     UNSPEC_ST1_SVE
 180     UNSPEC_LD1RQ
 181     UNSPEC_LD1_GATHER
 182     UNSPEC_ST1_SCATTER
 183     UNSPEC_MERGE_PTRUE
 184     UNSPEC_PTEST_PTRUE
 185     UNSPEC_UNPACKSHI
 186     UNSPEC_UNPACKUHI
 187     UNSPEC_UNPACKSLO
 188     UNSPEC_UNPACKULO
 189     UNSPEC_PACK
 190     UNSPEC_FLOAT_CONVERT
 191     UNSPEC_WHILE_LO
 192     UNSPEC_LDN
 193     UNSPEC_STN
 194     UNSPEC_INSR
 195     UNSPEC_CLASTB
 196     UNSPEC_FADDA
 197     UNSPEC_REV_SUBREG
 198 ])
 199
 200 (define_c_enum "unspecv" [
 201     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 202     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 203     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 204     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 205     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 206     UNSPECV_BLOCKAGE            ; Represent a blockage
 207     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 208   ]
 209 )
 210
 211 ;; If further include files are added the defintion of MD_INCLUDES
 212 ;; must be updated.
 213
 214 (include "constraints.md")
 215 (include "predicates.md")
 216 (include "iterators.md")
 217
 218 ;; -------------------------------------------------------------------
 219 ;; Instruction types and attributes
 220 ;; -------------------------------------------------------------------
 221
 222 ; The "type" attribute is included here from AArch32 backend to be able
 223 ; to share pipeline descriptions.
 224 (include "../arm/types.md")
 225
 226 ;; It is important to set the fp or simd attributes to yes when a pattern
 227 ;; alternative uses the FP or SIMD register files, usually signified by use of
 228 ;; the 'w' constraint.  This will ensure that the alternative will be
 229 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 230 ;; architecture extensions.  If all the alternatives in a pattern use the
 231 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 232 ;; or TARGET_SIMD.
 233
 234 ;; Attribute that specifies whether or not the instruction touches fp
 235 ;; registers.  When this is set to yes for an alternative, that alternative
 236 ;; will be disabled when !TARGET_FLOAT.
 237 (define_attr "fp" "no,yes" (const_string "no"))
 238
 239 ;; Attribute that specifies whether or not the instruction touches half
 240 ;; precision fp registers.  When this is set to yes for an alternative,
 241 ;; that alternative will be disabled when !TARGET_FP_F16INST.
 242 (define_attr "fp16" "no,yes" (const_string "no"))
 243
 244 ;; Attribute that specifies whether or not the instruction touches simd
 245 ;; registers.  When this is set to yes for an alternative, that alternative
 246 ;; will be disabled when !TARGET_SIMD.
 247 (define_attr "simd" "no,yes" (const_string "no"))
 248
 249 ;; Attribute that specifies whether or not the instruction uses SVE.
 250 ;; When this is set to yes for an alternative, that alternative
 251 ;; will be disabled when !TARGET_SVE.
 252 (define_attr "sve" "no,yes" (const_string "no"))
 253
 254 (define_attr "length" ""
 255   (const_int 4))
 256
 257 ;; Attribute that controls whether an alternative is enabled or not.
 258 ;; Currently it is only used to disable alternatives which touch fp or simd
 259 ;; registers when -mgeneral-regs-only is specified.
 260 (define_attr "enabled" "no,yes"
 261   (cond [(ior
 262             (and (eq_attr "fp" "yes")
 263                  (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
 264             (and (eq_attr "simd" "yes")
 265                  (eq (symbol_ref "TARGET_SIMD") (const_int 0)))
 266             (and (eq_attr "fp16" "yes")
 267                  (eq (symbol_ref "TARGET_FP_F16INST") (const_int 0)))
 268             (and (eq_attr "sve" "yes")
 269                  (eq (symbol_ref "TARGET_SVE") (const_int 0))))
 270             (const_string "no")
 271         ] (const_string "yes")))
 272
 273 ;; Attribute that specifies whether we are dealing with a branch to a
 274 ;; label that is far away, i.e. further away than the maximum/minimum
 275 ;; representable in a signed 21-bits number.
 276 ;; 0 :=: no
 277 ;; 1 :=: yes
 278 (define_attr "far_branch" "" (const_int 0))
 279
 280 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 281 ;; no predicated insns.
 282 (define_attr "predicated" "yes,no" (const_string "no"))
 283
 284 ;; -------------------------------------------------------------------
 285 ;; Pipeline descriptions and scheduling
 286 ;; -------------------------------------------------------------------
 287
 288 ;; Processor types.
 289 (include "aarch64-tune.md")
 290
 291 ;; Scheduling
 292 (include "../arm/cortex-a53.md")
 293 (include "../arm/cortex-a57.md")
 294 (include "../arm/exynos-m1.md")
 295 (include "falkor.md")
 296 (include "thunderx.md")
 297 (include "../arm/xgene1.md")
 298 (include "thunderx2t99.md")
 299
 300 ;; -------------------------------------------------------------------
 301 ;; Jumps and other miscellaneous insns
 302 ;; -------------------------------------------------------------------
 303
 304 (define_insn "indirect_jump"
 305   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 306   ""
 307   "br\\t%0"
 308   [(set_attr "type" "branch")]
 309 )
 310
 311 (define_insn "jump"
 312   [(set (pc) (label_ref (match_operand 0 "" "")))]
 313   ""
 314   "b\\t%l0"
 315   [(set_attr "type" "branch")]
 316 )
 317
 318 (define_expand "cbranch<mode>4"
 319   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 320                             [(match_operand:GPI 1 "register_operand" "")
 321                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 322                            (label_ref (match_operand 3 "" ""))
 323                            (pc)))]
 324   ""
 325   "
 326   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 327                                          operands[2]);
 328   operands[2] = const0_rtx;
 329   "
 330 )
 331
 332 (define_expand "cbranch<mode>4"
 333   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 334                             [(match_operand:GPF 1 "register_operand" "")
 335                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 336                            (label_ref (match_operand 3 "" ""))
 337                            (pc)))]
 338   ""
 339   "
 340   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 341                                          operands[2]);
 342   operands[2] = const0_rtx;
 343   "
 344 )
 345
 346 (define_expand "cbranchcc4"
 347   [(set (pc) (if_then_else
 348               (match_operator 0 "aarch64_comparison_operator"
 349                [(match_operand 1 "cc_register" "")
 350                 (match_operand 2 "const0_operand")])
 351               (label_ref (match_operand 3 "" ""))
 352               (pc)))]
 353   ""
 354   "")
 355
 356 (define_insn "ccmp<mode>"
 357   [(set (match_operand:CC 1 "cc_register" "")
 358         (if_then_else:CC
 359           (match_operator 4 "aarch64_comparison_operator"
 360            [(match_operand 0 "cc_register" "")
 361             (const_int 0)])
 362           (compare:CC
 363             (match_operand:GPI 2 "register_operand" "r,r,r")
 364             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 365           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 366   ""
 367   "@
 368    ccmp\\t%<w>2, %<w>3, %k5, %m4
 369    ccmp\\t%<w>2, %3, %k5, %m4
 370    ccmn\\t%<w>2, #%n3, %k5, %m4"
 371   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 372 )
 373
 374 (define_insn "fccmp<mode>"
 375   [(set (match_operand:CCFP 1 "cc_register" "")
 376         (if_then_else:CCFP
 377           (match_operator 4 "aarch64_comparison_operator"
 378            [(match_operand 0 "cc_register" "")
 379             (const_int 0)])
 380           (compare:CCFP
 381             (match_operand:GPF 2 "register_operand" "w")
 382             (match_operand:GPF 3 "register_operand" "w"))
 383           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 384   "TARGET_FLOAT"
 385   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 386   [(set_attr "type" "fccmp<s>")]
 387 )
 388
 389 (define_insn "fccmpe<mode>"
 390   [(set (match_operand:CCFPE 1 "cc_register" "")
 391          (if_then_else:CCFPE
 392           (match_operator 4 "aarch64_comparison_operator"
 393            [(match_operand 0 "cc_register" "")
 394           (const_int 0)])
 395            (compare:CCFPE
 396             (match_operand:GPF 2 "register_operand" "w")
 397             (match_operand:GPF 3 "register_operand" "w"))
 398           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 399   "TARGET_FLOAT"
 400   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 401   [(set_attr "type" "fccmp<s>")]
 402 )
 403
 404 ;; Expansion of signed mod by a power of 2 using CSNEG.
 405 ;; For x0 % n where n is a power of 2 produce:
 406 ;; negs   x1, x0
 407 ;; and    x0, x0, #(n - 1)
 408 ;; and    x1, x1, #(n - 1)
 409 ;; csneg  x0, x0, x1, mi
 410
 411 (define_expand "mod<mode>3"
 412   [(match_operand:GPI 0 "register_operand" "")
 413    (match_operand:GPI 1 "register_operand" "")
 414    (match_operand:GPI 2 "const_int_operand" "")]
 415   ""
 416   {
 417     HOST_WIDE_INT val = INTVAL (operands[2]);
 418
 419     if (val <= 0
 420        || exact_log2 (val) <= 0
 421        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 422       FAIL;
 423
 424     rtx mask = GEN_INT (val - 1);
 425
 426     /* In the special case of x0 % 2 we can do the even shorter:
 427         cmp    x0, xzr
 428         and    x0, x0, 1
 429         cneg   x0, x0, lt.  */
 430     if (val == 2)
 431       {
 432         rtx masked = gen_reg_rtx (<MODE>mode);
 433         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 434         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 435         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 436         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 437         DONE;
 438       }
 439
 440     rtx neg_op = gen_reg_rtx (<MODE>mode);
 441     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 442
 443     /* Extract the condition register and mode.  */
 444     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 445     rtx cc_reg = SET_DEST (cmp);
 446     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 447
 448     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 449     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 450
 451     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 452     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 453
 454     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 455                                        masked_neg, masked_pos));
 456     DONE;
 457   }
 458 )
 459
 460 (define_insn "condjump"
 461   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 462                             [(match_operand 1 "cc_register" "") (const_int 0)])
 463                            (label_ref (match_operand 2 "" ""))
 464                            (pc)))]
 465   ""
 466   {
 467     if (get_attr_length (insn) == 8)
 468       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 469     else
 470       return  "b%m0\\t%l2";
 471   }
 472   [(set_attr "type" "branch")
 473    (set (attr "length")
 474         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 475                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 476                       (const_int 4)
 477                       (const_int 8)))
 478    (set (attr "far_branch")
 479         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 480                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 481                       (const_int 0)
 482                       (const_int 1)))]
 483 )
 484
 485 ;; For a 24-bit immediate CST we can optimize the compare for equality
 486 ;; and branch sequence from:
 487 ;;      mov     x0, #imm1
 488 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 489 ;;      cmp     x1, x0
 490 ;;      b<ne,eq> .Label
 491 ;; into the shorter:
 492 ;;      sub     x0, x1, #(CST & 0xfff000)
 493 ;;      subs    x0, x0, #(CST & 0x000fff)
 494 ;;      b<ne,eq> .Label
 495 (define_insn_and_split "*compare_condjump<mode>"
 496   [(set (pc) (if_then_else (EQL
 497                               (match_operand:GPI 0 "register_operand" "r")
 498                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 499                            (label_ref:P (match_operand 2 "" ""))
 500                            (pc)))]
 501   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 502    && !aarch64_plus_operand (operands[1], <MODE>mode)
 503    && !reload_completed"
 504   "#"
 505   "&& true"
 506   [(const_int 0)]
 507   {
 508     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 509     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 510     rtx tmp = gen_reg_rtx (<MODE>mode);
 511     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 512     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 513     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 514     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 515     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 516     DONE;
 517   }
 518 )
 519
 520 (define_expand "casesi"
 521   [(match_operand:SI 0 "register_operand" "")   ; Index
 522    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 523    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 524    (match_operand:DI 3 "" "")                   ; Table label
 525    (match_operand:DI 4 "" "")]                  ; Out of range label
 526   ""
 527   {
 528     if (operands[1] != const0_rtx)
 529       {
 530         rtx reg = gen_reg_rtx (SImode);
 531
 532         /* Canonical RTL says that if you have:
 533
 534            (minus (X) (CONST))
 535
 536            then this should be emitted as:
 537
 538            (plus (X) (-CONST))
 539
 540            The use of trunc_int_for_mode ensures that the resulting
 541            constant can be represented in SImode, this is important
 542            for the corner case where operand[1] is INT_MIN.  */
 543
 544         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 545
 546         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 547               (operands[1], SImode))
 548           operands[1] = force_reg (SImode, operands[1]);
 549         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 550         operands[0] = reg;
 551       }
 552
 553     if (!aarch64_plus_operand (operands[2], SImode))
 554       operands[2] = force_reg (SImode, operands[2]);
 555     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 556                                                  const0_rtx),
 557                                     operands[0], operands[2], operands[4]));
 558
 559     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 560     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 561                                          operands[3]));
 562     DONE;
 563   }
 564 )
 565
 566 (define_insn "casesi_dispatch"
 567   [(parallel
 568     [(set (pc)
 569           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 570                            (match_operand:SI 1 "register_operand" "r")]
 571                         UNSPEC_CASESI)))
 572      (clobber (reg:CC CC_REGNUM))
 573      (clobber (match_scratch:DI 3 "=r"))
 574      (clobber (match_scratch:DI 4 "=r"))
 575      (use (label_ref (match_operand 2 "" "")))])]
 576   ""
 577   "*
 578   return aarch64_output_casesi (operands);
 579   "
 580   [(set_attr "length" "16")
 581    (set_attr "type" "branch")]
 582 )
 583
 584 (define_insn "nop"
 585   [(unspec[(const_int 0)] UNSPEC_NOP)]
 586   ""
 587   "nop"
 588   [(set_attr "type" "no_insn")]
 589 )
 590
 591 (define_insn "prefetch"
 592   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 593             (match_operand:QI 1 "const_int_operand" "")
 594             (match_operand:QI 2 "const_int_operand" ""))]
 595   ""
 596   {
 597     const char * pftype[2][4] =
 598     {
 599       {"prfm\\tPLDL1STRM, %0",
 600        "prfm\\tPLDL3KEEP, %0",
 601        "prfm\\tPLDL2KEEP, %0",
 602        "prfm\\tPLDL1KEEP, %0"},
 603       {"prfm\\tPSTL1STRM, %0",
 604        "prfm\\tPSTL3KEEP, %0",
 605        "prfm\\tPSTL2KEEP, %0",
 606        "prfm\\tPSTL1KEEP, %0"},
 607     };
 608
 609     int locality = INTVAL (operands[2]);
 610
 611     gcc_assert (IN_RANGE (locality, 0, 3));
 612
 613     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 614        the address into a DImode MEM so that aarch64_print_operand knows
 615        how to print it.  */
 616     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 617     return pftype[INTVAL(operands[1])][locality];
 618   }
 619   [(set_attr "type" "load_4")]
 620 )
 621
 622 (define_insn "trap"
 623   [(trap_if (const_int 1) (const_int 8))]
 624   ""
 625   "brk #1000"
 626   [(set_attr "type" "trap")])
 627
 628 (define_expand "prologue"
 629   [(clobber (const_int 0))]
 630   ""
 631   "
 632   aarch64_expand_prologue ();
 633   DONE;
 634   "
 635 )
 636
 637 (define_expand "epilogue"
 638   [(clobber (const_int 0))]
 639   ""
 640   "
 641   aarch64_expand_epilogue (false);
 642   DONE;
 643   "
 644 )
 645
 646 (define_expand "sibcall_epilogue"
 647   [(clobber (const_int 0))]
 648   ""
 649   "
 650   aarch64_expand_epilogue (true);
 651   DONE;
 652   "
 653 )
 654
 655 (define_insn "*do_return"
 656   [(return)]
 657   ""
 658   {
 659     if (aarch64_return_address_signing_enabled ()
 660         && TARGET_ARMV8_3
 661         && !crtl->calls_eh_return)
 662       return "retaa";
 663
 664     return "ret";
 665   }
 666   [(set_attr "type" "branch")]
 667 )
 668
 669 (define_expand "return"
 670   [(simple_return)]
 671   "aarch64_use_return_insn_p ()"
 672   ""
 673 )
 674
 675 (define_insn "simple_return"
 676   [(simple_return)]
 677   ""
 678   "ret"
 679   [(set_attr "type" "branch")]
 680 )
 681
 682 (define_insn "*cb<optab><mode>1"
 683   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 684                                 (const_int 0))
 685                            (label_ref (match_operand 1 "" ""))
 686                            (pc)))]
 687   ""
 688   {
 689     if (get_attr_length (insn) == 8)
 690       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 691     else
 692       return "<cbz>\\t%<w>0, %l1";
 693   }
 694   [(set_attr "type" "branch")
 695    (set (attr "length")
 696         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 697                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 698                       (const_int 4)
 699                       (const_int 8)))
 700    (set (attr "far_branch")
 701         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 702                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 703                       (const_int 0)
 704                       (const_int 1)))]
 705 )
 706
 707 (define_insn "*tb<optab><mode>1"
 708   [(set (pc) (if_then_else
 709               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 710                                     (const_int 1)
 711                                     (match_operand 1
 712                                       "aarch64_simd_shift_imm_<mode>" "n"))
 713                    (const_int 0))
 714              (label_ref (match_operand 2 "" ""))
 715              (pc)))
 716    (clobber (reg:CC CC_REGNUM))]
 717   ""
 718   {
 719     if (get_attr_length (insn) == 8)
 720       {
 721         if (get_attr_far_branch (insn) == 1)
 722           return aarch64_gen_far_branch (operands, 2, "Ltb",
 723                                          "<inv_tb>\\t%<w>0, %1, ");
 724         else
 725           {
 726             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 727             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 728           }
 729       }
 730     else
 731       return "<tbz>\t%<w>0, %1, %l2";
 732   }
 733   [(set_attr "type" "branch")
 734    (set (attr "length")
 735         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 736                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 737                       (const_int 4)
 738                       (const_int 8)))
 739    (set (attr "far_branch")
 740         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 741                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 742                       (const_int 0)
 743                       (const_int 1)))]
 744
 745 )
 746
 747 (define_insn "*cb<optab><mode>1"
 748   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 749                                  (const_int 0))
 750                            (label_ref (match_operand 1 "" ""))
 751                            (pc)))
 752    (clobber (reg:CC CC_REGNUM))]
 753   ""
 754   {
 755     if (get_attr_length (insn) == 8)
 756       {
 757         if (get_attr_far_branch (insn) == 1)
 758           return aarch64_gen_far_branch (operands, 1, "Ltb",
 759                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 760         else
 761           {
 762             char buf[64];
 763             uint64_t val = ((uint64_t) 1)
 764                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 765             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 766             output_asm_insn (buf, operands);
 767             return "<bcond>\t%l1";
 768           }
 769       }
 770     else
 771       return "<tbz>\t%<w>0, <sizem1>, %l1";
 772   }
 773   [(set_attr "type" "branch")
 774    (set (attr "length")
 775         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 776                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 777                       (const_int 4)
 778                       (const_int 8)))
 779    (set (attr "far_branch")
 780         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 781                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 782                       (const_int 0)
 783                       (const_int 1)))]
 784 )
 785
 786 ;; -------------------------------------------------------------------
 787 ;; Subroutine calls and sibcalls
 788 ;; -------------------------------------------------------------------
 789
 790 (define_expand "call"
 791   [(parallel [(call (match_operand 0 "memory_operand" "")
 792                     (match_operand 1 "general_operand" ""))
 793               (use (match_operand 2 "" ""))
 794               (clobber (reg:DI LR_REGNUM))])]
 795   ""
 796   "
 797   {
 798     aarch64_expand_call (NULL_RTX, operands[0], false);
 799     DONE;
 800   }"
 801 )
 802
 803 (define_insn "*call_insn"
 804   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 805          (match_operand 1 "" ""))
 806    (clobber (reg:DI LR_REGNUM))]
 807   ""
 808   "@
 809   blr\\t%0
 810   bl\\t%c0"
 811   [(set_attr "type" "call, call")]
 812 )
 813
 814 (define_expand "call_value"
 815   [(parallel [(set (match_operand 0 "" "")
 816                    (call (match_operand 1 "memory_operand" "")
 817                          (match_operand 2 "general_operand" "")))
 818               (use (match_operand 3 "" ""))
 819               (clobber (reg:DI LR_REGNUM))])]
 820   ""
 821   "
 822   {
 823     aarch64_expand_call (operands[0], operands[1], false);
 824     DONE;
 825   }"
 826 )
 827
 828 (define_insn "*call_value_insn"
 829   [(set (match_operand 0 "" "")
 830         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 831                       (match_operand 2 "" "")))
 832    (clobber (reg:DI LR_REGNUM))]
 833   ""
 834   "@
 835   blr\\t%1
 836   bl\\t%c1"
 837   [(set_attr "type" "call, call")]
 838 )
 839
 840 (define_expand "sibcall"
 841   [(parallel [(call (match_operand 0 "memory_operand" "")
 842                     (match_operand 1 "general_operand" ""))
 843               (return)
 844               (use (match_operand 2 "" ""))])]
 845   ""
 846   {
 847     aarch64_expand_call (NULL_RTX, operands[0], true);
 848     DONE;
 849   }
 850 )
 851
 852 (define_expand "sibcall_value"
 853   [(parallel [(set (match_operand 0 "" "")
 854                    (call (match_operand 1 "memory_operand" "")
 855                          (match_operand 2 "general_operand" "")))
 856               (return)
 857               (use (match_operand 3 "" ""))])]
 858   ""
 859   {
 860     aarch64_expand_call (operands[0], operands[1], true);
 861     DONE;
 862   }
 863 )
 864
 865 (define_insn "*sibcall_insn"
 866   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 867          (match_operand 1 "" ""))
 868    (return)]
 869   "SIBLING_CALL_P (insn)"
 870   "@
 871    br\\t%0
 872    b\\t%c0"
 873   [(set_attr "type" "branch, branch")]
 874 )
 875
 876 (define_insn "*sibcall_value_insn"
 877   [(set (match_operand 0 "" "")
 878         (call (mem:DI
 879                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 880               (match_operand 2 "" "")))
 881    (return)]
 882   "SIBLING_CALL_P (insn)"
 883   "@
 884    br\\t%1
 885    b\\t%c1"
 886   [(set_attr "type" "branch, branch")]
 887 )
 888
 889 ;; Call subroutine returning any type.
 890
 891 (define_expand "untyped_call"
 892   [(parallel [(call (match_operand 0 "")
 893                     (const_int 0))
 894               (match_operand 1 "")
 895               (match_operand 2 "")])]
 896   ""
 897 {
 898   int i;
 899
 900   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 901
 902   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 903     {
 904       rtx set = XVECEXP (operands[2], 0, i);
 905       emit_move_insn (SET_DEST (set), SET_SRC (set));
 906     }
 907
 908   /* The optimizer does not know that the call sets the function value
 909      registers we stored in the result block.  We avoid problems by
 910      claiming that all hard registers are used and clobbered at this
 911      point.  */
 912   emit_insn (gen_blockage ());
 913   DONE;
 914 })
 915
 916 ;; -------------------------------------------------------------------
 917 ;; Moves
 918 ;; -------------------------------------------------------------------
 919
 920 (define_expand "mov<mode>"
 921   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 922         (match_operand:SHORT 1 "general_operand" ""))]
 923   ""
 924   "
 925     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 926       operands[1] = force_reg (<MODE>mode, operands[1]);
 927
 928     if (GET_CODE (operands[1]) == CONST_POLY_INT)
 929       {
 930         aarch64_expand_mov_immediate (operands[0], operands[1]);
 931         DONE;
 932       }
 933   "
 934 )
 935
 936 (define_insn "*mov<mode>_aarch64"
 937   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
 938         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
 939   "(register_operand (operands[0], <MODE>mode)
 940     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 941 {
 942    switch (which_alternative)
 943      {
 944      case 0:
 945        return "mov\t%w0, %w1";
 946      case 1:
 947        return "mov\t%w0, %1";
 948      case 2:
 949        return aarch64_output_scalar_simd_mov_immediate (operands[1],
 950                                                         <MODE>mode);
 951      case 3:
 952        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
 953      case 4:
 954        return "ldr<size>\t%w0, %1";
 955      case 5:
 956        return "ldr\t%<size>0, %1";
 957      case 6:
 958        return "str<size>\t%w1, %0";
 959      case 7:
 960        return "str\t%<size>1, %0";
 961      case 8:
 962        return "umov\t%w0, %1.<v>[0]";
 963      case 9:
 964        return "dup\t%0.<Vallxd>, %w1";
 965      case 10:
 966        return "dup\t%<Vetype>0, %1.<v>[0]";
 967      default:
 968        gcc_unreachable ();
 969      }
 970 }
 971   ;; The "mov_imm" type for CNT is just a placeholder.
 972   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
 973                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
 974    (set_attr "simd" "*,*,yes,*,*,*,*,*,yes,yes,yes")
 975    (set_attr "sve" "*,*,*,yes,*,*,*,*,*,*,*")]
 976 )
 977
 978 (define_expand "mov<mode>"
 979   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
 980         (match_operand:GPI 1 "general_operand" ""))]
 981   ""
 982   "
 983     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
 984         && <MODE>mode == DImode
 985         && aarch64_split_dimode_const_store (operands[0], operands[1]))
 986       DONE;
 987
 988     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 989       operands[1] = force_reg (<MODE>mode, operands[1]);
 990
 991     /* FIXME: RR we still need to fix up what we are doing with
 992        symbol_refs and other types of constants.  */
 993     if (CONSTANT_P (operands[1])
 994         && !CONST_INT_P (operands[1]))
 995      {
 996        aarch64_expand_mov_immediate (operands[0], operands[1]);
 997        DONE;
 998      }
 999   "
1000 )
1001
1002 (define_insn_and_split "*movsi_aarch64"
1003   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1004         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1005   "(register_operand (operands[0], SImode)
1006     || aarch64_reg_or_zero (operands[1], SImode))"
1007   "@
1008    mov\\t%w0, %w1
1009    mov\\t%w0, %w1
1010    mov\\t%w0, %w1
1011    mov\\t%w0, %1
1012    #
1013    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1014    ldr\\t%w0, %1
1015    ldr\\t%s0, %1
1016    str\\t%w1, %0
1017    str\\t%s1, %0
1018    adr\\t%x0, %c1
1019    adrp\\t%x0, %A1
1020    fmov\\t%s0, %w1
1021    fmov\\t%w0, %s1
1022    fmov\\t%s0, %s1
1023    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1024   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1025     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1026    [(const_int 0)]
1027    "{
1028        aarch64_expand_mov_immediate (operands[0], operands[1]);
1029        DONE;
1030     }"
1031   ;; The "mov_imm" type for CNT is just a placeholder.
1032   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1033                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1034    (set_attr "fp" "*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1035    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1036    (set_attr "sve" "*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1037 )
1038
1039 (define_insn_and_split "*movdi_aarch64"
1040   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1041         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1042   "(register_operand (operands[0], DImode)
1043     || aarch64_reg_or_zero (operands[1], DImode))"
1044   "@
1045    mov\\t%x0, %x1
1046    mov\\t%0, %x1
1047    mov\\t%x0, %1
1048    mov\\t%x0, %1
1049    mov\\t%w0, %1
1050    #
1051    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1052    ldr\\t%x0, %1
1053    ldr\\t%d0, %1
1054    str\\t%x1, %0
1055    str\\t%d1, %0
1056    adr\\t%x0, %c1
1057    adrp\\t%x0, %A1
1058    fmov\\t%d0, %x1
1059    fmov\\t%x0, %d1
1060    fmov\\t%d0, %d1
1061    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1062    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1063     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1064    [(const_int 0)]
1065    "{
1066        aarch64_expand_mov_immediate (operands[0], operands[1]);
1067        DONE;
1068     }"
1069   ;; The "mov_imm" type for CNTD is just a placeholder.
1070   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1071                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1072                      neon_move")
1073    (set_attr "fp" "*,*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1074    (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1075    (set_attr "sve" "*,*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1076 )
1077
1078 (define_insn "insv_imm<mode>"
1079   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1080                           (const_int 16)
1081                           (match_operand:GPI 1 "const_int_operand" "n"))
1082         (match_operand:GPI 2 "const_int_operand" "n"))]
1083   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1084    && UINTVAL (operands[1]) % 16 == 0"
1085   "movk\\t%<w>0, %X2, lsl %1"
1086   [(set_attr "type" "mov_imm")]
1087 )
1088
1089 (define_expand "movti"
1090   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1091         (match_operand:TI 1 "general_operand" ""))]
1092   ""
1093   "
1094     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1095       operands[1] = force_reg (TImode, operands[1]);
1096
1097     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1098       {
1099         emit_move_insn (gen_lowpart (DImode, operands[0]),
1100                         gen_lowpart (DImode, operands[1]));
1101         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1102         DONE;
1103       }
1104   "
1105 )
1106
1107 (define_insn "*movti_aarch64"
1108   [(set (match_operand:TI 0
1109          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1110         (match_operand:TI 1
1111          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1112   "(register_operand (operands[0], TImode)
1113     || aarch64_reg_or_zero (operands[1], TImode))"
1114   "@
1115    #
1116    #
1117    #
1118    mov\\t%0.16b, %1.16b
1119    ldp\\t%0, %H0, %1
1120    stp\\t%1, %H1, %0
1121    stp\\txzr, xzr, %0
1122    ldr\\t%q0, %1
1123    str\\t%q1, %0"
1124   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1125                              load_16,store_16,store_16,\
1126                              load_16,store_16")
1127    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1128    (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1129    (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1130 )
1131
1132 ;; Split a TImode register-register or register-immediate move into
1133 ;; its component DImode pieces, taking care to handle overlapping
1134 ;; source and dest registers.
1135 (define_split
1136    [(set (match_operand:TI 0 "register_operand" "")
1137          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1138   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1139   [(const_int 0)]
1140 {
1141   aarch64_split_128bit_move (operands[0], operands[1]);
1142   DONE;
1143 })
1144
1145 (define_expand "mov<mode>"
1146   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1147         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1148   ""
1149   {
1150     if (!TARGET_FLOAT)
1151       {
1152         aarch64_err_no_fpadvsimd (<MODE>mode, "code");
1153         FAIL;
1154       }
1155
1156     if (GET_CODE (operands[0]) == MEM
1157         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1158               && aarch64_float_const_zero_rtx_p (operands[1])))
1159       operands[1] = force_reg (<MODE>mode, operands[1]);
1160   }
1161 )
1162
1163 (define_insn "*movhf_aarch64"
1164   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r")
1165         (match_operand:HF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1166   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1167     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1168   "@
1169    movi\\t%0.4h, #0
1170    fmov\\t%h0, %w1
1171    umov\\t%w0, %1.h[0]
1172    mov\\t%0.h[0], %1.h[0]
1173    fmov\\t%h0, %1
1174    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1175    ldr\\t%h0, %1
1176    str\\t%h1, %0
1177    ldrh\\t%w0, %1
1178    strh\\t%w1, %0
1179    mov\\t%w0, %w1"
1180   [(set_attr "type" "neon_move,f_mcr,neon_to_gp,neon_move,fconsts, \
1181                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1182    (set_attr "simd" "yes,*,yes,yes,*,yes,*,*,*,*,*")
1183    (set_attr "fp16"   "*,yes,*,*,yes,*,*,*,*,*,*")]
1184 )
1185
1186 (define_insn "*movsf_aarch64"
1187   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1188         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1189   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1190     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1191   "@
1192    movi\\t%0.2s, #0
1193    fmov\\t%s0, %w1
1194    fmov\\t%w0, %s1
1195    fmov\\t%s0, %s1
1196    fmov\\t%s0, %1
1197    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1198    ldr\\t%s0, %1
1199    str\\t%s1, %0
1200    ldr\\t%w0, %1
1201    str\\t%w1, %0
1202    mov\\t%w0, %w1
1203    mov\\t%w0, %1"
1204   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1205                      f_loads,f_stores,load_4,store_4,mov_reg,\
1206                      fconsts")
1207    (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1208 )
1209
1210 (define_insn "*movdf_aarch64"
1211   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1212         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1213   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1214     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1215   "@
1216    movi\\t%d0, #0
1217    fmov\\t%d0, %x1
1218    fmov\\t%x0, %d1
1219    fmov\\t%d0, %d1
1220    fmov\\t%d0, %1
1221    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1222    ldr\\t%d0, %1
1223    str\\t%d1, %0
1224    ldr\\t%x0, %1
1225    str\\t%x1, %0
1226    mov\\t%x0, %x1
1227    mov\\t%x0, %1"
1228   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1229                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1230                      fconstd")
1231    (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1232 )
1233
1234 (define_split
1235   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1236         (match_operand:GPF_HF 1 "general_operand"))]
1237   "can_create_pseudo_p ()
1238    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1239    && !aarch64_float_const_representable_p (operands[1])
1240    &&  aarch64_float_const_rtx_p (operands[1])"
1241   [(const_int 0)]
1242   {
1243     unsigned HOST_WIDE_INT ival;
1244     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1245       FAIL;
1246
1247     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1248     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1249     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1250     DONE;
1251   }
1252 )
1253
1254 (define_insn "*movtf_aarch64"
1255   [(set (match_operand:TF 0
1256          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1257         (match_operand:TF 1
1258          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1259   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1260     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1261   "@
1262    mov\\t%0.16b, %1.16b
1263    #
1264    #
1265    #
1266    movi\\t%0.2d, #0
1267    fmov\\t%s0, wzr
1268    ldr\\t%q0, %1
1269    str\\t%q1, %0
1270    ldp\\t%0, %H0, %1
1271    stp\\t%1, %H1, %0
1272    stp\\txzr, xzr, %0"
1273   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1274                      f_loadd,f_stored,load_16,store_16,store_16")
1275    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1276    (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1277 )
1278
1279 (define_split
1280    [(set (match_operand:TF 0 "register_operand" "")
1281          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1282   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1283   [(const_int 0)]
1284   {
1285     aarch64_split_128bit_move (operands[0], operands[1]);
1286     DONE;
1287   }
1288 )
1289
1290 ;; 0 is dst
1291 ;; 1 is src
1292 ;; 2 is size of move in bytes
1293 ;; 3 is alignment
1294
1295 (define_expand "movmemdi"
1296   [(match_operand:BLK 0 "memory_operand")
1297    (match_operand:BLK 1 "memory_operand")
1298    (match_operand:DI 2 "immediate_operand")
1299    (match_operand:DI 3 "immediate_operand")]
1300    "!STRICT_ALIGNMENT"
1301 {
1302   if (aarch64_expand_movmem (operands))
1303     DONE;
1304   FAIL;
1305 }
1306 )
1307
1308 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1309 ;; fairly lax checking on the second memory operation.
1310 (define_insn "load_pairsi"
1311   [(set (match_operand:SI 0 "register_operand" "=r,w")
1312         (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1313    (set (match_operand:SI 2 "register_operand" "=r,w")
1314         (match_operand:SI 3 "memory_operand" "m,m"))]
1315   "rtx_equal_p (XEXP (operands[3], 0),
1316                 plus_constant (Pmode,
1317                                XEXP (operands[1], 0),
1318                                GET_MODE_SIZE (SImode)))"
1319   "@
1320    ldp\\t%w0, %w2, %1
1321    ldp\\t%s0, %s2, %1"
1322   [(set_attr "type" "load_8,neon_load1_2reg")
1323    (set_attr "fp" "*,yes")]
1324 )
1325
1326 (define_insn "load_pairdi"
1327   [(set (match_operand:DI 0 "register_operand" "=r,w")
1328         (match_operand:DI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1329    (set (match_operand:DI 2 "register_operand" "=r,w")
1330         (match_operand:DI 3 "memory_operand" "m,m"))]
1331   "rtx_equal_p (XEXP (operands[3], 0),
1332                 plus_constant (Pmode,
1333                                XEXP (operands[1], 0),
1334                                GET_MODE_SIZE (DImode)))"
1335   "@
1336    ldp\\t%x0, %x2, %1
1337    ldp\\t%d0, %d2, %1"
1338   [(set_attr "type" "load_16,neon_load1_2reg")
1339    (set_attr "fp" "*,yes")]
1340 )
1341
1342
1343 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1344 ;; fairly lax checking on the second memory operation.
1345 (define_insn "store_pairsi"
1346   [(set (match_operand:SI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1347         (match_operand:SI 1 "aarch64_reg_or_zero" "rZ,w"))
1348    (set (match_operand:SI 2 "memory_operand" "=m,m")
1349         (match_operand:SI 3 "aarch64_reg_or_zero" "rZ,w"))]
1350   "rtx_equal_p (XEXP (operands[2], 0),
1351                 plus_constant (Pmode,
1352                                XEXP (operands[0], 0),
1353                                GET_MODE_SIZE (SImode)))"
1354   "@
1355    stp\\t%w1, %w3, %0
1356    stp\\t%s1, %s3, %0"
1357   [(set_attr "type" "store_8,neon_store1_2reg")
1358    (set_attr "fp" "*,yes")]
1359 )
1360
1361 (define_insn "store_pairdi"
1362   [(set (match_operand:DI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1363         (match_operand:DI 1 "aarch64_reg_or_zero" "rZ,w"))
1364    (set (match_operand:DI 2 "memory_operand" "=m,m")
1365         (match_operand:DI 3 "aarch64_reg_or_zero" "rZ,w"))]
1366   "rtx_equal_p (XEXP (operands[2], 0),
1367                 plus_constant (Pmode,
1368                                XEXP (operands[0], 0),
1369                                GET_MODE_SIZE (DImode)))"
1370   "@
1371    stp\\t%x1, %x3, %0
1372    stp\\t%d1, %d3, %0"
1373   [(set_attr "type" "store_16,neon_store1_2reg")
1374    (set_attr "fp" "*,yes")]
1375 )
1376
1377 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1378 ;; fairly lax checking on the second memory operation.
1379 (define_insn "load_pairsf"
1380   [(set (match_operand:SF 0 "register_operand" "=w,r")
1381         (match_operand:SF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1382    (set (match_operand:SF 2 "register_operand" "=w,r")
1383         (match_operand:SF 3 "memory_operand" "m,m"))]
1384   "rtx_equal_p (XEXP (operands[3], 0),
1385                 plus_constant (Pmode,
1386                                XEXP (operands[1], 0),
1387                                GET_MODE_SIZE (SFmode)))"
1388   "@
1389    ldp\\t%s0, %s2, %1
1390    ldp\\t%w0, %w2, %1"
1391   [(set_attr "type" "neon_load1_2reg,load_8")
1392    (set_attr "fp" "yes,*")]
1393 )
1394
1395 (define_insn "load_pairdf"
1396   [(set (match_operand:DF 0 "register_operand" "=w,r")
1397         (match_operand:DF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1398    (set (match_operand:DF 2 "register_operand" "=w,r")
1399         (match_operand:DF 3 "memory_operand" "m,m"))]
1400   "rtx_equal_p (XEXP (operands[3], 0),
1401                 plus_constant (Pmode,
1402                                XEXP (operands[1], 0),
1403                                GET_MODE_SIZE (DFmode)))"
1404   "@
1405    ldp\\t%d0, %d2, %1
1406    ldp\\t%x0, %x2, %1"
1407   [(set_attr "type" "neon_load1_2reg,load_16")
1408    (set_attr "fp" "yes,*")]
1409 )
1410
1411 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1412 ;; fairly lax checking on the second memory operation.
1413 (define_insn "store_pairsf"
1414   [(set (match_operand:SF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1415         (match_operand:SF 1 "aarch64_reg_or_fp_zero" "w,rY"))
1416    (set (match_operand:SF 2 "memory_operand" "=m,m")
1417         (match_operand:SF 3 "aarch64_reg_or_fp_zero" "w,rY"))]
1418   "rtx_equal_p (XEXP (operands[2], 0),
1419                 plus_constant (Pmode,
1420                                XEXP (operands[0], 0),
1421                                GET_MODE_SIZE (SFmode)))"
1422   "@
1423    stp\\t%s1, %s3, %0
1424    stp\\t%w1, %w3, %0"
1425   [(set_attr "type" "neon_store1_2reg,store_8")
1426    (set_attr "fp" "yes,*")]
1427 )
1428
1429 (define_insn "store_pairdf"
1430   [(set (match_operand:DF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1431         (match_operand:DF 1 "aarch64_reg_or_fp_zero" "w,rY"))
1432    (set (match_operand:DF 2 "memory_operand" "=m,m")
1433         (match_operand:DF 3 "aarch64_reg_or_fp_zero" "w,rY"))]
1434   "rtx_equal_p (XEXP (operands[2], 0),
1435                 plus_constant (Pmode,
1436                                XEXP (operands[0], 0),
1437                                GET_MODE_SIZE (DFmode)))"
1438   "@
1439    stp\\t%d1, %d3, %0
1440    stp\\t%x1, %x3, %0"
1441   [(set_attr "type" "neon_store1_2reg,store_16")
1442    (set_attr "fp" "yes,*")]
1443 )
1444
1445 ;; Load pair with post-index writeback.  This is primarily used in function
1446 ;; epilogues.
1447 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1448   [(parallel
1449     [(set (match_operand:P 0 "register_operand" "=k")
1450           (plus:P (match_operand:P 1 "register_operand" "0")
1451                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1452      (set (match_operand:GPI 2 "register_operand" "=r")
1453           (mem:GPI (match_dup 1)))
1454      (set (match_operand:GPI 3 "register_operand" "=r")
1455           (mem:GPI (plus:P (match_dup 1)
1456                    (match_operand:P 5 "const_int_operand" "n"))))])]
1457   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1458   "ldp\\t%<w>2, %<w>3, [%1], %4"
1459   [(set_attr "type" "load_<ldpstp_sz>")]
1460 )
1461
1462 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1463   [(parallel
1464     [(set (match_operand:P 0 "register_operand" "=k")
1465           (plus:P (match_operand:P 1 "register_operand" "0")
1466                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1467      (set (match_operand:GPF 2 "register_operand" "=w")
1468           (mem:GPF (match_dup 1)))
1469      (set (match_operand:GPF 3 "register_operand" "=w")
1470           (mem:GPF (plus:P (match_dup 1)
1471                    (match_operand:P 5 "const_int_operand" "n"))))])]
1472   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1473   "ldp\\t%<w>2, %<w>3, [%1], %4"
1474   [(set_attr "type" "neon_load1_2reg")]
1475 )
1476
1477 ;; Store pair with pre-index writeback.  This is primarily used in function
1478 ;; prologues.
1479 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1480   [(parallel
1481     [(set (match_operand:P 0 "register_operand" "=&k")
1482           (plus:P (match_operand:P 1 "register_operand" "0")
1483                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1484      (set (mem:GPI (plus:P (match_dup 0)
1485                    (match_dup 4)))
1486           (match_operand:GPI 2 "register_operand" "r"))
1487      (set (mem:GPI (plus:P (match_dup 0)
1488                    (match_operand:P 5 "const_int_operand" "n")))
1489           (match_operand:GPI 3 "register_operand" "r"))])]
1490   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1491   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1492   [(set_attr "type" "store_<ldpstp_sz>")]
1493 )
1494
1495 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1496   [(parallel
1497     [(set (match_operand:P 0 "register_operand" "=&k")
1498           (plus:P (match_operand:P 1 "register_operand" "0")
1499                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1500      (set (mem:GPF (plus:P (match_dup 0)
1501                    (match_dup 4)))
1502           (match_operand:GPF 2 "register_operand" "w"))
1503      (set (mem:GPF (plus:P (match_dup 0)
1504                    (match_operand:P 5 "const_int_operand" "n")))
1505           (match_operand:GPF 3 "register_operand" "w"))])]
1506   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1507   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1508   [(set_attr "type" "neon_store1_2reg<q>")]
1509 )
1510
1511 ;; -------------------------------------------------------------------
1512 ;; Sign/Zero extension
1513 ;; -------------------------------------------------------------------
1514
1515 (define_expand "<optab>sidi2"
1516   [(set (match_operand:DI 0 "register_operand")
1517         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1518   ""
1519 )
1520
1521 (define_insn "*extendsidi2_aarch64"
1522   [(set (match_operand:DI 0 "register_operand" "=r,r")
1523         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1524   ""
1525   "@
1526    sxtw\t%0, %w1
1527    ldrsw\t%0, %1"
1528   [(set_attr "type" "extend,load_4")]
1529 )
1530
1531 (define_insn "*load_pair_extendsidi2_aarch64"
1532   [(set (match_operand:DI 0 "register_operand" "=r")
1533         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1534    (set (match_operand:DI 2 "register_operand" "=r")
1535         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1536   "rtx_equal_p (XEXP (operands[3], 0),
1537                 plus_constant (Pmode,
1538                                XEXP (operands[1], 0),
1539                                GET_MODE_SIZE (SImode)))"
1540   "ldpsw\\t%0, %2, %1"
1541   [(set_attr "type" "load_8")]
1542 )
1543
1544 (define_insn "*zero_extendsidi2_aarch64"
1545   [(set (match_operand:DI 0 "register_operand" "=r,r")
1546         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1547   ""
1548   "@
1549    uxtw\t%0, %w1
1550    ldr\t%w0, %1"
1551   [(set_attr "type" "extend,load_4")]
1552 )
1553
1554 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1555   [(set (match_operand:DI 0 "register_operand" "=r")
1556         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1557    (set (match_operand:DI 2 "register_operand" "=r")
1558         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1559   "rtx_equal_p (XEXP (operands[3], 0),
1560                 plus_constant (Pmode,
1561                                XEXP (operands[1], 0),
1562                                GET_MODE_SIZE (SImode)))"
1563   "ldp\\t%w0, %w2, %1"
1564   [(set_attr "type" "load_8")]
1565 )
1566
1567 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1568   [(set (match_operand:GPI 0 "register_operand")
1569         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1570   ""
1571 )
1572
1573 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1574   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1575         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1576   ""
1577   "@
1578    sxt<SHORT:size>\t%<GPI:w>0, %w1
1579    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1580   [(set_attr "type" "extend,load_4")]
1581 )
1582
1583 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1584   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1585         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1586   ""
1587   "@
1588    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1589    ldr<SHORT:size>\t%w0, %1
1590    ldr\t%<SHORT:size>0, %1"
1591   [(set_attr "type" "logic_imm,load_4,load_4")]
1592 )
1593
1594 (define_expand "<optab>qihi2"
1595   [(set (match_operand:HI 0 "register_operand")
1596         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1597   ""
1598 )
1599
1600 (define_insn "*extendqihi2_aarch64"
1601   [(set (match_operand:HI 0 "register_operand" "=r,r")
1602         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1603   ""
1604   "@
1605    sxtb\t%w0, %w1
1606    ldrsb\t%w0, %1"
1607   [(set_attr "type" "extend,load_4")]
1608 )
1609
1610 (define_insn "*zero_extendqihi2_aarch64"
1611   [(set (match_operand:HI 0 "register_operand" "=r,r")
1612         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1613   ""
1614   "@
1615    and\t%w0, %w1, 255
1616    ldrb\t%w0, %1"
1617   [(set_attr "type" "logic_imm,load_4")]
1618 )
1619
1620 ;; -------------------------------------------------------------------
1621 ;; Simple arithmetic
1622 ;; -------------------------------------------------------------------
1623
1624 (define_expand "add<mode>3"
1625   [(set
1626     (match_operand:GPI 0 "register_operand" "")
1627     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1628               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1629   ""
1630 {
1631   /* If operands[1] is a subreg extract the inner RTX.  */
1632   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1633
1634   /* If the constant is too large for a single instruction and isn't frame
1635      based, split off the immediate so it is available for CSE.  */
1636   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1637       && can_create_pseudo_p ()
1638       && (!REG_P (op1)
1639          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1640     operands[2] = force_reg (<MODE>mode, operands[2]);
1641   /* Expand polynomial additions now if the destination is the stack
1642      pointer, since we don't want to use that as a temporary.  */
1643   else if (operands[0] == stack_pointer_rtx
1644            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1645     {
1646       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1647                                 operands[2], NULL_RTX, NULL_RTX);
1648       DONE;
1649     }
1650 })
1651
1652 (define_insn "*add<mode>3_aarch64"
1653   [(set
1654     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1655     (plus:GPI
1656      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1657      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1658   ""
1659   "@
1660   add\\t%<w>0, %<w>1, %2
1661   add\\t%<w>0, %<w>1, %<w>2
1662   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1663   sub\\t%<w>0, %<w>1, #%n2
1664   #
1665   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1666   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1667   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1668    (set_attr "simd" "*,*,yes,*,*,*")]
1669 )
1670
1671 ;; zero_extend version of above
1672 (define_insn "*addsi3_aarch64_uxtw"
1673   [(set
1674     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1675     (zero_extend:DI
1676      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1677               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1678   ""
1679   "@
1680   add\\t%w0, %w1, %2
1681   add\\t%w0, %w1, %w2
1682   sub\\t%w0, %w1, #%n2
1683   #"
1684   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1685 )
1686
1687 ;; If there's a free register, and we can load the constant with a
1688 ;; single instruction, do so.  This has a chance to improve scheduling.
1689 (define_peephole2
1690   [(match_scratch:GPI 3 "r")
1691    (set (match_operand:GPI 0 "register_operand")
1692         (plus:GPI
1693           (match_operand:GPI 1 "register_operand")
1694           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1695   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1696   [(set (match_dup 3) (match_dup 2))
1697    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1698 )
1699
1700 (define_peephole2
1701   [(match_scratch:SI 3 "r")
1702    (set (match_operand:DI 0 "register_operand")
1703         (zero_extend:DI
1704           (plus:SI
1705             (match_operand:SI 1 "register_operand")
1706             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1707   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1708   [(set (match_dup 3) (match_dup 2))
1709    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1710 )
1711
1712 ;; After peephole2 has had a chance to run, split any remaining long
1713 ;; additions into two add immediates.
1714 (define_split
1715   [(set (match_operand:GPI 0 "register_operand")
1716         (plus:GPI
1717           (match_operand:GPI 1 "register_operand")
1718           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1719   "epilogue_completed"
1720   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1721    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1722   {
1723     HOST_WIDE_INT i = INTVAL (operands[2]);
1724     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1725     operands[3] = GEN_INT (i - s);
1726     operands[4] = GEN_INT (s);
1727   }
1728 )
1729
1730 ;; Match addition of polynomial offsets that require one temporary, for which
1731 ;; we can use the early-clobbered destination register.  This is a separate
1732 ;; pattern so that the early clobber doesn't affect register allocation
1733 ;; for other forms of addition.  However, we still need to provide an
1734 ;; all-register alternative, in case the offset goes out of range after
1735 ;; elimination.  For completeness we might as well provide all GPR-based
1736 ;; alternatives from the main pattern.
1737 ;;
1738 ;; We don't have a pattern for additions requiring two temporaries since at
1739 ;; present LRA doesn't allow new scratches to be added during elimination.
1740 ;; Such offsets should be rare anyway.
1741 ;;
1742 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1743 ;; here would go away.  We could just handle all polynomial constants in
1744 ;; this pattern.
1745 (define_insn_and_split "*add<mode>3_poly_1"
1746   [(set
1747     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1748     (plus:GPI
1749      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1750      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1751   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1752   "@
1753   add\\t%<w>0, %<w>1, %2
1754   add\\t%<w>0, %<w>1, %<w>2
1755   sub\\t%<w>0, %<w>1, #%n2
1756   #
1757   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1758   #"
1759   "&& epilogue_completed
1760    && !reg_overlap_mentioned_p (operands[0], operands[1])
1761    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1762   [(const_int 0)]
1763   {
1764     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1765                               operands[2], operands[0], NULL_RTX);
1766     DONE;
1767   }
1768   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1769   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1770 )
1771
1772 (define_split
1773   [(set (match_operand:DI 0 "register_operand")
1774         (zero_extend:DI
1775           (plus:SI
1776             (match_operand:SI 1 "register_operand")
1777             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1778   "epilogue_completed"
1779   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1780    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1781   {
1782     HOST_WIDE_INT i = INTVAL (operands[2]);
1783     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1784     operands[3] = GEN_INT (i - s);
1785     operands[4] = GEN_INT (s);
1786     operands[5] = gen_lowpart (SImode, operands[0]);
1787   }
1788 )
1789
1790 (define_expand "addti3"
1791   [(set (match_operand:TI 0 "register_operand" "")
1792         (plus:TI (match_operand:TI 1 "register_operand" "")
1793                  (match_operand:TI 2 "register_operand" "")))]
1794   ""
1795 {
1796   rtx low = gen_reg_rtx (DImode);
1797   emit_insn (gen_adddi3_compareC (low, gen_lowpart (DImode, operands[1]),
1798                                   gen_lowpart (DImode, operands[2])));
1799
1800   rtx high = gen_reg_rtx (DImode);
1801   emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1802                                  gen_highpart (DImode, operands[2])));
1803
1804   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1805   emit_move_insn (gen_highpart (DImode, operands[0]), high);
1806   DONE;
1807 })
1808
1809 (define_insn "add<mode>3_compare0"
1810   [(set (reg:CC_NZ CC_REGNUM)
1811         (compare:CC_NZ
1812          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1813                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1814          (const_int 0)))
1815    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1816         (plus:GPI (match_dup 1) (match_dup 2)))]
1817   ""
1818   "@
1819   adds\\t%<w>0, %<w>1, %<w>2
1820   adds\\t%<w>0, %<w>1, %2
1821   subs\\t%<w>0, %<w>1, #%n2"
1822   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1823 )
1824
1825 ;; zero_extend version of above
1826 (define_insn "*addsi3_compare0_uxtw"
1827   [(set (reg:CC_NZ CC_REGNUM)
1828         (compare:CC_NZ
1829          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1830                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1831          (const_int 0)))
1832    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1833         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1834   ""
1835   "@
1836   adds\\t%w0, %w1, %w2
1837   adds\\t%w0, %w1, %2
1838   subs\\t%w0, %w1, #%n2"
1839   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1840 )
1841
1842 (define_insn "*add<mode>3_compareC_cconly_imm"
1843   [(set (reg:CC_C CC_REGNUM)
1844         (ne:CC_C
1845           (plus:<DWI>
1846             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1847             (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1848           (zero_extend:<DWI>
1849             (plus:GPI
1850               (match_dup 0)
1851               (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1852   "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1853                                  <MODE>mode, operands[1])"
1854   "@
1855   cmn\\t%<w>0, %1
1856   cmp\\t%<w>0, #%n1"
1857   [(set_attr "type" "alus_imm")]
1858 )
1859
1860 (define_insn "*add<mode>3_compareC_cconly"
1861   [(set (reg:CC_C CC_REGNUM)
1862         (ne:CC_C
1863           (plus:<DWI>
1864             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1865             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1866           (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1867   ""
1868   "cmn\\t%<w>0, %<w>1"
1869   [(set_attr "type" "alus_sreg")]
1870 )
1871
1872 (define_insn "*add<mode>3_compareC_imm"
1873   [(set (reg:CC_C CC_REGNUM)
1874         (ne:CC_C
1875           (plus:<DWI>
1876             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1877             (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1878           (zero_extend:<DWI>
1879             (plus:GPI
1880               (match_dup 1)
1881               (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1882    (set (match_operand:GPI 0 "register_operand" "=r,r")
1883         (plus:GPI (match_dup 1) (match_dup 2)))]
1884   "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1885                                  <MODE>mode, operands[2])"
1886   "@
1887   adds\\t%<w>0, %<w>1, %2
1888   subs\\t%<w>0, %<w>1, #%n2"
1889   [(set_attr "type" "alus_imm")]
1890 )
1891
1892 (define_insn "add<mode>3_compareC"
1893   [(set (reg:CC_C CC_REGNUM)
1894         (ne:CC_C
1895           (plus:<DWI>
1896             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1897             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1898           (zero_extend:<DWI>
1899             (plus:GPI (match_dup 1) (match_dup 2)))))
1900    (set (match_operand:GPI 0 "register_operand" "=r")
1901         (plus:GPI (match_dup 1) (match_dup 2)))]
1902   ""
1903   "adds\\t%<w>0, %<w>1, %<w>2"
1904   [(set_attr "type" "alus_sreg")]
1905 )
1906
1907 (define_insn "*adds_shift_imm_<mode>"
1908   [(set (reg:CC_NZ CC_REGNUM)
1909         (compare:CC_NZ
1910          (plus:GPI (ASHIFT:GPI
1911                     (match_operand:GPI 1 "register_operand" "r")
1912                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1913                    (match_operand:GPI 3 "register_operand" "r"))
1914          (const_int 0)))
1915    (set (match_operand:GPI 0 "register_operand" "=r")
1916         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1917                   (match_dup 3)))]
1918   ""
1919   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1920   [(set_attr "type" "alus_shift_imm")]
1921 )
1922
1923 (define_insn "*subs_shift_imm_<mode>"
1924   [(set (reg:CC_NZ CC_REGNUM)
1925         (compare:CC_NZ
1926          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1927                     (ASHIFT:GPI
1928                      (match_operand:GPI 2 "register_operand" "r")
1929                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1930          (const_int 0)))
1931    (set (match_operand:GPI 0 "register_operand" "=r")
1932         (minus:GPI (match_dup 1)
1933                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1934   ""
1935   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1936   [(set_attr "type" "alus_shift_imm")]
1937 )
1938
1939 (define_insn "*adds_mul_imm_<mode>"
1940   [(set (reg:CC_NZ CC_REGNUM)
1941         (compare:CC_NZ
1942          (plus:GPI (mult:GPI
1943                     (match_operand:GPI 1 "register_operand" "r")
1944                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1945                    (match_operand:GPI 3 "register_operand" "r"))
1946          (const_int 0)))
1947    (set (match_operand:GPI 0 "register_operand" "=r")
1948         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1949                   (match_dup 3)))]
1950   ""
1951   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1952   [(set_attr "type" "alus_shift_imm")]
1953 )
1954
1955 (define_insn "*subs_mul_imm_<mode>"
1956   [(set (reg:CC_NZ CC_REGNUM)
1957         (compare:CC_NZ
1958          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1959                     (mult:GPI
1960                      (match_operand:GPI 2 "register_operand" "r")
1961                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1962          (const_int 0)))
1963    (set (match_operand:GPI 0 "register_operand" "=r")
1964         (minus:GPI (match_dup 1)
1965                    (mult:GPI (match_dup 2) (match_dup 3))))]
1966   ""
1967   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1968   [(set_attr "type" "alus_shift_imm")]
1969 )
1970
1971 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1972   [(set (reg:CC_NZ CC_REGNUM)
1973         (compare:CC_NZ
1974          (plus:GPI
1975           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1976           (match_operand:GPI 2 "register_operand" "r"))
1977         (const_int 0)))
1978    (set (match_operand:GPI 0 "register_operand" "=r")
1979         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1980   ""
1981   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1982   [(set_attr "type" "alus_ext")]
1983 )
1984
1985 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1986   [(set (reg:CC_NZ CC_REGNUM)
1987         (compare:CC_NZ
1988          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1989                     (ANY_EXTEND:GPI
1990                      (match_operand:ALLX 2 "register_operand" "r")))
1991         (const_int 0)))
1992    (set (match_operand:GPI 0 "register_operand" "=r")
1993         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1994   ""
1995   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1996   [(set_attr "type" "alus_ext")]
1997 )
1998
1999 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2000   [(set (reg:CC_NZ CC_REGNUM)
2001         (compare:CC_NZ
2002          (plus:GPI (ashift:GPI
2003                     (ANY_EXTEND:GPI
2004                      (match_operand:ALLX 1 "register_operand" "r"))
2005                     (match_operand 2 "aarch64_imm3" "Ui3"))
2006                    (match_operand:GPI 3 "register_operand" "r"))
2007          (const_int 0)))
2008    (set (match_operand:GPI 0 "register_operand" "=rk")
2009         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2010                               (match_dup 2))
2011                   (match_dup 3)))]
2012   ""
2013   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2014   [(set_attr "type" "alus_ext")]
2015 )
2016
2017 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2018   [(set (reg:CC_NZ CC_REGNUM)
2019         (compare:CC_NZ
2020          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2021                     (ashift:GPI
2022                      (ANY_EXTEND:GPI
2023                       (match_operand:ALLX 2 "register_operand" "r"))
2024                      (match_operand 3 "aarch64_imm3" "Ui3")))
2025          (const_int 0)))
2026    (set (match_operand:GPI 0 "register_operand" "=rk")
2027         (minus:GPI (match_dup 1)
2028                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2029                                (match_dup 3))))]
2030   ""
2031   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2032   [(set_attr "type" "alus_ext")]
2033 )
2034
2035 (define_insn "*adds_<optab><mode>_multp2"
2036   [(set (reg:CC_NZ CC_REGNUM)
2037         (compare:CC_NZ
2038          (plus:GPI (ANY_EXTRACT:GPI
2039                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2040                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2041                     (match_operand 3 "const_int_operand" "n")
2042                     (const_int 0))
2043                    (match_operand:GPI 4 "register_operand" "r"))
2044         (const_int 0)))
2045    (set (match_operand:GPI 0 "register_operand" "=r")
2046         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2047                                    (match_dup 3)
2048                                    (const_int 0))
2049                   (match_dup 4)))]
2050   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2051   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2052   [(set_attr "type" "alus_ext")]
2053 )
2054
2055 (define_insn "*subs_<optab><mode>_multp2"
2056   [(set (reg:CC_NZ CC_REGNUM)
2057         (compare:CC_NZ
2058          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
2059                     (ANY_EXTRACT:GPI
2060                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2061                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2062                      (match_operand 3 "const_int_operand" "n")
2063                      (const_int 0)))
2064         (const_int 0)))
2065    (set (match_operand:GPI 0 "register_operand" "=r")
2066         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2067                                   (mult:GPI (match_dup 1) (match_dup 2))
2068                                   (match_dup 3)
2069                                   (const_int 0))))]
2070   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2071   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2072   [(set_attr "type" "alus_ext")]
2073 )
2074
2075 (define_insn "*add<mode>3nr_compare0"
2076   [(set (reg:CC_NZ CC_REGNUM)
2077         (compare:CC_NZ
2078          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2079                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2080          (const_int 0)))]
2081   ""
2082   "@
2083   cmn\\t%<w>0, %<w>1
2084   cmn\\t%<w>0, %1
2085   cmp\\t%<w>0, #%n1"
2086   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2087 )
2088
2089 (define_insn "aarch64_sub<mode>_compare0"
2090   [(set (reg:CC_NZ CC_REGNUM)
2091         (compare:CC_NZ
2092          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2093                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2094          (const_int 0)))]
2095   ""
2096   "cmp\\t%<w>0, %<w>1"
2097   [(set_attr "type" "alus_sreg")]
2098 )
2099
2100 (define_insn "*compare_neg<mode>"
2101   [(set (reg:CC_Z CC_REGNUM)
2102         (compare:CC_Z
2103          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2104          (match_operand:GPI 1 "register_operand" "r")))]
2105   ""
2106   "cmn\\t%<w>1, %<w>0"
2107   [(set_attr "type" "alus_sreg")]
2108 )
2109
2110 (define_insn "*add_<shift>_<mode>"
2111   [(set (match_operand:GPI 0 "register_operand" "=r")
2112         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2113                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2114                   (match_operand:GPI 3 "register_operand" "r")))]
2115   ""
2116   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2117   [(set_attr "type" "alu_shift_imm")]
2118 )
2119
2120 ;; zero_extend version of above
2121 (define_insn "*add_<shift>_si_uxtw"
2122   [(set (match_operand:DI 0 "register_operand" "=r")
2123         (zero_extend:DI
2124          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2125                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2126                   (match_operand:SI 3 "register_operand" "r"))))]
2127   ""
2128   "add\\t%w0, %w3, %w1, <shift> %2"
2129   [(set_attr "type" "alu_shift_imm")]
2130 )
2131
2132 (define_insn "*add_mul_imm_<mode>"
2133   [(set (match_operand:GPI 0 "register_operand" "=r")
2134         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2135                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2136                   (match_operand:GPI 3 "register_operand" "r")))]
2137   ""
2138   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2139   [(set_attr "type" "alu_shift_imm")]
2140 )
2141
2142 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2143   [(set (match_operand:GPI 0 "register_operand" "=rk")
2144         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2145                   (match_operand:GPI 2 "register_operand" "r")))]
2146   ""
2147   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2148   [(set_attr "type" "alu_ext")]
2149 )
2150
2151 ;; zero_extend version of above
2152 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2153   [(set (match_operand:DI 0 "register_operand" "=rk")
2154         (zero_extend:DI
2155          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2156                   (match_operand:GPI 2 "register_operand" "r"))))]
2157   ""
2158   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2159   [(set_attr "type" "alu_ext")]
2160 )
2161
2162 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2163   [(set (match_operand:GPI 0 "register_operand" "=rk")
2164         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2165                                (match_operand:ALLX 1 "register_operand" "r"))
2166                               (match_operand 2 "aarch64_imm3" "Ui3"))
2167                   (match_operand:GPI 3 "register_operand" "r")))]
2168   ""
2169   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2170   [(set_attr "type" "alu_ext")]
2171 )
2172
2173 ;; zero_extend version of above
2174 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2175   [(set (match_operand:DI 0 "register_operand" "=rk")
2176         (zero_extend:DI
2177          (plus:SI (ashift:SI (ANY_EXTEND:SI
2178                               (match_operand:SHORT 1 "register_operand" "r"))
2179                              (match_operand 2 "aarch64_imm3" "Ui3"))
2180                   (match_operand:SI 3 "register_operand" "r"))))]
2181   ""
2182   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2183   [(set_attr "type" "alu_ext")]
2184 )
2185
2186 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2187   [(set (match_operand:GPI 0 "register_operand" "=rk")
2188         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2189                              (match_operand:ALLX 1 "register_operand" "r"))
2190                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2191                   (match_operand:GPI 3 "register_operand" "r")))]
2192   ""
2193   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2194   [(set_attr "type" "alu_ext")]
2195 )
2196
2197 ;; zero_extend version of above
2198 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2199   [(set (match_operand:DI 0 "register_operand" "=rk")
2200         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2201                              (match_operand:SHORT 1 "register_operand" "r"))
2202                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2203                   (match_operand:SI 3 "register_operand" "r"))))]
2204   ""
2205   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2206   [(set_attr "type" "alu_ext")]
2207 )
2208
2209 (define_insn "*add_<optab><mode>_multp2"
2210   [(set (match_operand:GPI 0 "register_operand" "=rk")
2211         (plus:GPI (ANY_EXTRACT:GPI
2212                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2213                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2214                    (match_operand 3 "const_int_operand" "n")
2215                    (const_int 0))
2216                   (match_operand:GPI 4 "register_operand" "r")))]
2217   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2218   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2219   [(set_attr "type" "alu_ext")]
2220 )
2221
2222 ;; zero_extend version of above
2223 (define_insn "*add_<optab>si_multp2_uxtw"
2224   [(set (match_operand:DI 0 "register_operand" "=rk")
2225         (zero_extend:DI
2226          (plus:SI (ANY_EXTRACT:SI
2227                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2228                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2229                    (match_operand 3 "const_int_operand" "n")
2230                    (const_int 0))
2231                   (match_operand:SI 4 "register_operand" "r"))))]
2232   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2233   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2234   [(set_attr "type" "alu_ext")]
2235 )
2236
2237 (define_expand "add<mode>3_carryin"
2238   [(set (match_operand:GPI 0 "register_operand")
2239         (plus:GPI
2240           (plus:GPI
2241             (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2242             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2243           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2244    ""
2245    ""
2246 )
2247
2248 ;; Note that add with carry with two zero inputs is matched by cset,
2249 ;; and that add with carry with one zero input is matched by cinc.
2250
2251 (define_insn "*add<mode>3_carryin"
2252   [(set (match_operand:GPI 0 "register_operand" "=r")
2253         (plus:GPI
2254           (plus:GPI
2255             (match_operand:GPI 3 "aarch64_carry_operation" "")
2256             (match_operand:GPI 1 "register_operand" "r"))
2257           (match_operand:GPI 2 "register_operand" "r")))]
2258    ""
2259    "adc\\t%<w>0, %<w>1, %<w>2"
2260   [(set_attr "type" "adc_reg")]
2261 )
2262
2263 ;; zero_extend version of above
2264 (define_insn "*addsi3_carryin_uxtw"
2265   [(set (match_operand:DI 0 "register_operand" "=r")
2266         (zero_extend:DI
2267           (plus:SI
2268             (plus:SI
2269               (match_operand:SI 3 "aarch64_carry_operation" "")
2270               (match_operand:SI 1 "register_operand" "r"))
2271             (match_operand:SI 2 "register_operand" "r"))))]
2272    ""
2273    "adc\\t%w0, %w1, %w2"
2274   [(set_attr "type" "adc_reg")]
2275 )
2276
2277 (define_insn "*add_uxt<mode>_shift2"
2278   [(set (match_operand:GPI 0 "register_operand" "=rk")
2279         (plus:GPI (and:GPI
2280                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2281                                (match_operand 2 "aarch64_imm3" "Ui3"))
2282                    (match_operand 3 "const_int_operand" "n"))
2283                   (match_operand:GPI 4 "register_operand" "r")))]
2284   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2285   "*
2286   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2287                                            INTVAL (operands[3])));
2288   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2289   [(set_attr "type" "alu_ext")]
2290 )
2291
2292 ;; zero_extend version of above
2293 (define_insn "*add_uxtsi_shift2_uxtw"
2294   [(set (match_operand:DI 0 "register_operand" "=rk")
2295         (zero_extend:DI
2296          (plus:SI (and:SI
2297                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2298                               (match_operand 2 "aarch64_imm3" "Ui3"))
2299                    (match_operand 3 "const_int_operand" "n"))
2300                   (match_operand:SI 4 "register_operand" "r"))))]
2301   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2302   "*
2303   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2304                                            INTVAL (operands[3])));
2305   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2306   [(set_attr "type" "alu_ext")]
2307 )
2308
2309 (define_insn "*add_uxt<mode>_multp2"
2310   [(set (match_operand:GPI 0 "register_operand" "=rk")
2311         (plus:GPI (and:GPI
2312                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2313                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2314                    (match_operand 3 "const_int_operand" "n"))
2315                   (match_operand:GPI 4 "register_operand" "r")))]
2316   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2317   "*
2318   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2319                                            INTVAL (operands[3])));
2320   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2321   [(set_attr "type" "alu_ext")]
2322 )
2323
2324 ;; zero_extend version of above
2325 (define_insn "*add_uxtsi_multp2_uxtw"
2326   [(set (match_operand:DI 0 "register_operand" "=rk")
2327         (zero_extend:DI
2328          (plus:SI (and:SI
2329                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2330                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2331                    (match_operand 3 "const_int_operand" "n"))
2332                   (match_operand:SI 4 "register_operand" "r"))))]
2333   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2334   "*
2335   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2336                                            INTVAL (operands[3])));
2337   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2338   [(set_attr "type" "alu_ext")]
2339 )
2340
2341 (define_insn "subsi3"
2342   [(set (match_operand:SI 0 "register_operand" "=rk")
2343         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2344                   (match_operand:SI 2 "register_operand" "r")))]
2345   ""
2346   "sub\\t%w0, %w1, %w2"
2347   [(set_attr "type" "alu_sreg")]
2348 )
2349
2350 ;; zero_extend version of above
2351 (define_insn "*subsi3_uxtw"
2352   [(set (match_operand:DI 0 "register_operand" "=rk")
2353         (zero_extend:DI
2354          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2355                    (match_operand:SI 2 "register_operand" "r"))))]
2356   ""
2357   "sub\\t%w0, %w1, %w2"
2358   [(set_attr "type" "alu_sreg")]
2359 )
2360
2361 (define_insn "subdi3"
2362   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2363         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2364                   (match_operand:DI 2 "register_operand" "r,w")))]
2365   ""
2366   "@
2367    sub\\t%x0, %x1, %x2
2368    sub\\t%d0, %d1, %d2"
2369   [(set_attr "type" "alu_sreg, neon_sub")
2370    (set_attr "simd" "*,yes")]
2371 )
2372
2373 (define_expand "subti3"
2374   [(set (match_operand:TI 0 "register_operand" "")
2375         (minus:TI (match_operand:TI 1 "register_operand" "")
2376                   (match_operand:TI 2 "register_operand" "")))]
2377   ""
2378 {
2379   rtx low = gen_reg_rtx (DImode);
2380   emit_insn (gen_subdi3_compare1 (low, gen_lowpart (DImode, operands[1]),
2381                                   gen_lowpart (DImode, operands[2])));
2382
2383   rtx high = gen_reg_rtx (DImode);
2384   emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2385                                  gen_highpart (DImode, operands[2])));
2386
2387   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2388   emit_move_insn (gen_highpart (DImode, operands[0]), high);
2389   DONE;
2390 })
2391
2392 (define_insn "*sub<mode>3_compare0"
2393   [(set (reg:CC_NZ CC_REGNUM)
2394         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2395                                   (match_operand:GPI 2 "register_operand" "r"))
2396                        (const_int 0)))
2397    (set (match_operand:GPI 0 "register_operand" "=r")
2398         (minus:GPI (match_dup 1) (match_dup 2)))]
2399   ""
2400   "subs\\t%<w>0, %<w>1, %<w>2"
2401   [(set_attr "type" "alus_sreg")]
2402 )
2403
2404 ;; zero_extend version of above
2405 (define_insn "*subsi3_compare0_uxtw"
2406   [(set (reg:CC_NZ CC_REGNUM)
2407         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2408                                  (match_operand:SI 2 "register_operand" "r"))
2409                        (const_int 0)))
2410    (set (match_operand:DI 0 "register_operand" "=r")
2411         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2412   ""
2413   "subs\\t%w0, %w1, %w2"
2414   [(set_attr "type" "alus_sreg")]
2415 )
2416
2417 (define_insn "sub<mode>3_compare1"
2418   [(set (reg:CC CC_REGNUM)
2419         (compare:CC
2420           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2421           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2422    (set (match_operand:GPI 0 "register_operand" "=r")
2423         (minus:GPI (match_dup 1) (match_dup 2)))]
2424   ""
2425   "subs\\t%<w>0, %<w>1, %<w>2"
2426   [(set_attr "type" "alus_sreg")]
2427 )
2428
2429 (define_insn "sub<mode>3_compare1_imm"
2430   [(set (reg:CC CC_REGNUM)
2431         (compare:CC
2432           (match_operand:GPI 1 "register_operand" "r")
2433           (match_operand:GPI 3 "const_int_operand" "n")))
2434    (set (match_operand:GPI 0 "register_operand" "=r")
2435         (plus:GPI (match_dup 1)
2436                   (match_operand:GPI 2 "aarch64_sub_immediate" "J")))]
2437   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2438   "subs\\t%<w>0, %<w>1, #%n2"
2439   [(set_attr "type" "alus_sreg")]
2440 )
2441
2442 (define_peephole2
2443   [(set (match_operand:GPI 0 "register_operand")
2444         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
2445                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
2446    (set (reg:CC CC_REGNUM)
2447         (compare:CC
2448           (match_dup 1)
2449           (match_dup 2)))]
2450   "!reg_overlap_mentioned_p (operands[0], operands[1])
2451    && !reg_overlap_mentioned_p (operands[0], operands[2])"
2452   [(const_int 0)]
2453   {
2454     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2455                                          operands[2]));
2456     DONE;
2457   }
2458 )
2459
2460 ;; Same as the above peephole but with the compare and minus in
2461 ;; swapped order.  The restriction on overlap between operand 0
2462 ;; and operands 1 and 2 doesn't apply here.
2463 (define_peephole2
2464   [(set (reg:CC CC_REGNUM)
2465         (compare:CC
2466           (match_operand:GPI 1 "aarch64_reg_or_zero")
2467           (match_operand:GPI 2 "aarch64_reg_or_zero")))
2468    (set (match_operand:GPI 0 "register_operand")
2469         (minus:GPI (match_dup 1)
2470                    (match_dup 2)))]
2471   ""
2472   [(const_int 0)]
2473   {
2474     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2475                                          operands[2]));
2476     DONE;
2477   }
2478 )
2479
2480 (define_peephole2
2481   [(set (match_operand:GPI 0 "register_operand")
2482         (plus:GPI (match_operand:GPI 1 "register_operand")
2483                   (match_operand:GPI 2 "aarch64_sub_immediate")))
2484    (set (reg:CC CC_REGNUM)
2485         (compare:CC
2486           (match_dup 1)
2487           (match_operand:GPI 3 "const_int_operand")))]
2488   "!reg_overlap_mentioned_p (operands[0], operands[1])
2489    && INTVAL (operands[3]) == -INTVAL (operands[2])"
2490   [(const_int 0)]
2491   {
2492     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2493                                          operands[2], operands[3]));
2494     DONE;
2495   }
2496 )
2497
2498 ;; Same as the above peephole but with the compare and minus in
2499 ;; swapped order.  The restriction on overlap between operand 0
2500 ;; and operands 1 doesn't apply here.
2501 (define_peephole2
2502   [(set (reg:CC CC_REGNUM)
2503         (compare:CC
2504           (match_operand:GPI 1 "register_operand")
2505           (match_operand:GPI 3 "const_int_operand")))
2506    (set (match_operand:GPI 0 "register_operand")
2507         (plus:GPI (match_dup 1)
2508                   (match_operand:GPI 2 "aarch64_sub_immediate")))]
2509   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2510   [(const_int 0)]
2511   {
2512     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2513                                          operands[2], operands[3]));
2514     DONE;
2515   }
2516 )
2517
2518 (define_insn "*sub_<shift>_<mode>"
2519   [(set (match_operand:GPI 0 "register_operand" "=r")
2520         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2521                    (ASHIFT:GPI
2522                     (match_operand:GPI 1 "register_operand" "r")
2523                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2524   ""
2525   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2526   [(set_attr "type" "alu_shift_imm")]
2527 )
2528
2529 ;; zero_extend version of above
2530 (define_insn "*sub_<shift>_si_uxtw"
2531   [(set (match_operand:DI 0 "register_operand" "=r")
2532         (zero_extend:DI
2533          (minus:SI (match_operand:SI 3 "register_operand" "r")
2534                    (ASHIFT:SI
2535                     (match_operand:SI 1 "register_operand" "r")
2536                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2537   ""
2538   "sub\\t%w0, %w3, %w1, <shift> %2"
2539   [(set_attr "type" "alu_shift_imm")]
2540 )
2541
2542 (define_insn "*sub_mul_imm_<mode>"
2543   [(set (match_operand:GPI 0 "register_operand" "=r")
2544         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2545                    (mult:GPI
2546                     (match_operand:GPI 1 "register_operand" "r")
2547                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2548   ""
2549   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2550   [(set_attr "type" "alu_shift_imm")]
2551 )
2552
2553 ;; zero_extend version of above
2554 (define_insn "*sub_mul_imm_si_uxtw"
2555   [(set (match_operand:DI 0 "register_operand" "=r")
2556         (zero_extend:DI
2557          (minus:SI (match_operand:SI 3 "register_operand" "r")
2558                    (mult:SI
2559                     (match_operand:SI 1 "register_operand" "r")
2560                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2561   ""
2562   "sub\\t%w0, %w3, %w1, lsl %p2"
2563   [(set_attr "type" "alu_shift_imm")]
2564 )
2565
2566 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2567   [(set (match_operand:GPI 0 "register_operand" "=rk")
2568         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2569                    (ANY_EXTEND:GPI
2570                     (match_operand:ALLX 2 "register_operand" "r"))))]
2571   ""
2572   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2573   [(set_attr "type" "alu_ext")]
2574 )
2575
2576 ;; zero_extend version of above
2577 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2578   [(set (match_operand:DI 0 "register_operand" "=rk")
2579         (zero_extend:DI
2580          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2581                    (ANY_EXTEND:SI
2582                     (match_operand:SHORT 2 "register_operand" "r")))))]
2583   ""
2584   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2585   [(set_attr "type" "alu_ext")]
2586 )
2587
2588 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2589   [(set (match_operand:GPI 0 "register_operand" "=rk")
2590         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2591                    (ashift:GPI (ANY_EXTEND:GPI
2592                                 (match_operand:ALLX 2 "register_operand" "r"))
2593                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2594   ""
2595   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2596   [(set_attr "type" "alu_ext")]
2597 )
2598
2599 ;; zero_extend version of above
2600 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2601   [(set (match_operand:DI 0 "register_operand" "=rk")
2602         (zero_extend:DI
2603          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2604                    (ashift:SI (ANY_EXTEND:SI
2605                                (match_operand:SHORT 2 "register_operand" "r"))
2606                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2607   ""
2608   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2609   [(set_attr "type" "alu_ext")]
2610 )
2611
2612 (define_insn "*sub_<optab><mode>_multp2"
2613   [(set (match_operand:GPI 0 "register_operand" "=rk")
2614         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2615                    (ANY_EXTRACT:GPI
2616                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2617                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2618                     (match_operand 3 "const_int_operand" "n")
2619                     (const_int 0))))]
2620   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2621   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2622   [(set_attr "type" "alu_ext")]
2623 )
2624
2625 ;; zero_extend version of above
2626 (define_insn "*sub_<optab>si_multp2_uxtw"
2627   [(set (match_operand:DI 0 "register_operand" "=rk")
2628         (zero_extend:DI
2629          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2630                    (ANY_EXTRACT:SI
2631                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2632                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2633                     (match_operand 3 "const_int_operand" "n")
2634                     (const_int 0)))))]
2635   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2636   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2637   [(set_attr "type" "alu_ext")]
2638 )
2639
2640 ;; The hardware description is op1 + ~op2 + C.
2641 ;;                           = op1 + (-op2 + 1) + (1 - !C)
2642 ;;                           = op1 - op2 - 1 + 1 - !C
2643 ;;                           = op1 - op2 - !C.
2644 ;; We describe the latter.
2645
2646 (define_insn "*sub<mode>3_carryin0"
2647   [(set (match_operand:GPI 0 "register_operand" "=r")
2648         (minus:GPI
2649           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2650           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
2651    ""
2652    "sbc\\t%<w>0, %<w>1, <w>zr"
2653   [(set_attr "type" "adc_reg")]
2654 )
2655
2656 ;; zero_extend version of the above
2657 (define_insn "*subsi3_carryin_uxtw"
2658   [(set (match_operand:DI 0 "register_operand" "=r")
2659         (zero_extend:DI
2660           (minus:SI
2661             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2662             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
2663    ""
2664    "sbc\\t%w0, %w1, wzr"
2665   [(set_attr "type" "adc_reg")]
2666 )
2667
2668 (define_expand "sub<mode>3_carryin"
2669   [(set (match_operand:GPI 0 "register_operand")
2670         (minus:GPI
2671           (minus:GPI
2672             (match_operand:GPI 1 "aarch64_reg_or_zero")
2673             (match_operand:GPI 2 "register_operand"))
2674           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
2675    ""
2676    ""
2677 )
2678
2679 (define_insn "*sub<mode>3_carryin"
2680   [(set (match_operand:GPI 0 "register_operand" "=r")
2681         (minus:GPI
2682           (minus:GPI
2683             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2684             (match_operand:GPI 2 "register_operand" "r"))
2685           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
2686
2687    ""
2688    "sbc\\t%<w>0, %<w>1, %<w>2"
2689   [(set_attr "type" "adc_reg")]
2690 )
2691
2692 ;; zero_extend version of the above
2693 (define_insn "*subsi3_carryin_uxtw"
2694   [(set (match_operand:DI 0 "register_operand" "=r")
2695         (zero_extend:DI
2696           (minus:SI
2697             (minus:SI
2698               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2699               (match_operand:SI 2 "register_operand" "r"))
2700             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
2701
2702    ""
2703    "sbc\\t%w0, %w1, %w2"
2704   [(set_attr "type" "adc_reg")]
2705 )
2706
2707 (define_insn "*sub<mode>3_carryin_alt"
2708   [(set (match_operand:GPI 0 "register_operand" "=r")
2709         (minus:GPI
2710           (minus:GPI
2711             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2712             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
2713           (match_operand:GPI 2 "register_operand" "r")))]
2714    ""
2715    "sbc\\t%<w>0, %<w>1, %<w>2"
2716   [(set_attr "type" "adc_reg")]
2717 )
2718
2719 ;; zero_extend version of the above
2720 (define_insn "*subsi3_carryin_alt_uxtw"
2721   [(set (match_operand:DI 0 "register_operand" "=r")
2722         (zero_extend:DI
2723           (minus:SI
2724             (minus:SI
2725               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2726               (match_operand:SI 3 "aarch64_borrow_operation" ""))
2727             (match_operand:SI 2 "register_operand" "r"))))]
2728    ""
2729    "sbc\\t%w0, %w1, %w2"
2730   [(set_attr "type" "adc_reg")]
2731 )
2732
2733 (define_insn "*sub_uxt<mode>_shift2"
2734   [(set (match_operand:GPI 0 "register_operand" "=rk")
2735         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2736                    (and:GPI
2737                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2738                                 (match_operand 2 "aarch64_imm3" "Ui3"))
2739                     (match_operand 3 "const_int_operand" "n"))))]
2740   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2741   "*
2742   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2743                                            INTVAL (operands[3])));
2744   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2745   [(set_attr "type" "alu_ext")]
2746 )
2747
2748 ;; zero_extend version of above
2749 (define_insn "*sub_uxtsi_shift2_uxtw"
2750   [(set (match_operand:DI 0 "register_operand" "=rk")
2751         (zero_extend:DI
2752          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2753                    (and:SI
2754                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
2755                                (match_operand 2 "aarch64_imm3" "Ui3"))
2756                     (match_operand 3 "const_int_operand" "n")))))]
2757   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2758   "*
2759   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2760                                            INTVAL (operands[3])));
2761   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2762   [(set_attr "type" "alu_ext")]
2763 )
2764
2765 (define_insn "*sub_uxt<mode>_multp2"
2766   [(set (match_operand:GPI 0 "register_operand" "=rk")
2767         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2768                    (and:GPI
2769                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2770                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2771                     (match_operand 3 "const_int_operand" "n"))))]
2772   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2773   "*
2774   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2775                                            INTVAL (operands[3])));
2776   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2777   [(set_attr "type" "alu_ext")]
2778 )
2779
2780 ;; zero_extend version of above
2781 (define_insn "*sub_uxtsi_multp2_uxtw"
2782   [(set (match_operand:DI 0 "register_operand" "=rk")
2783         (zero_extend:DI
2784          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2785                    (and:SI
2786                     (mult:SI (match_operand:SI 1 "register_operand" "r")
2787                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2788                     (match_operand 3 "const_int_operand" "n")))))]
2789   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2790   "*
2791   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2792                                            INTVAL (operands[3])));
2793   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2794   [(set_attr "type" "alu_ext")]
2795 )
2796
2797 (define_expand "abs<mode>2"
2798   [(match_operand:GPI 0 "register_operand" "")
2799    (match_operand:GPI 1 "register_operand" "")]
2800   ""
2801   {
2802     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2803     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2804     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2805     DONE;
2806   }
2807 )
2808
2809 (define_insn "neg<mode>2"
2810   [(set (match_operand:GPI 0 "register_operand" "=r,w")
2811         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2812   ""
2813   "@
2814    neg\\t%<w>0, %<w>1
2815    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2816   [(set_attr "type" "alu_sreg, neon_neg<q>")
2817    (set_attr "simd" "*,yes")]
2818 )
2819
2820 ;; zero_extend version of above
2821 (define_insn "*negsi2_uxtw"
2822   [(set (match_operand:DI 0 "register_operand" "=r")
2823         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2824   ""
2825   "neg\\t%w0, %w1"
2826   [(set_attr "type" "alu_sreg")]
2827 )
2828
2829 (define_insn "*ngc<mode>"
2830   [(set (match_operand:GPI 0 "register_operand" "=r")
2831         (minus:GPI
2832           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
2833           (match_operand:GPI 1 "register_operand" "r")))]
2834   ""
2835   "ngc\\t%<w>0, %<w>1"
2836   [(set_attr "type" "adc_reg")]
2837 )
2838
2839 (define_insn "*ngcsi_uxtw"
2840   [(set (match_operand:DI 0 "register_operand" "=r")
2841         (zero_extend:DI
2842           (minus:SI
2843             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
2844             (match_operand:SI 1 "register_operand" "r"))))]
2845   ""
2846   "ngc\\t%w0, %w1"
2847   [(set_attr "type" "adc_reg")]
2848 )
2849
2850 (define_insn "neg<mode>2_compare0"
2851   [(set (reg:CC_NZ CC_REGNUM)
2852         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2853                        (const_int 0)))
2854    (set (match_operand:GPI 0 "register_operand" "=r")
2855         (neg:GPI (match_dup 1)))]
2856   ""
2857   "negs\\t%<w>0, %<w>1"
2858   [(set_attr "type" "alus_sreg")]
2859 )
2860
2861 ;; zero_extend version of above
2862 (define_insn "*negsi2_compare0_uxtw"
2863   [(set (reg:CC_NZ CC_REGNUM)
2864         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2865                        (const_int 0)))
2866    (set (match_operand:DI 0 "register_operand" "=r")
2867         (zero_extend:DI (neg:SI (match_dup 1))))]
2868   ""
2869   "negs\\t%w0, %w1"
2870   [(set_attr "type" "alus_sreg")]
2871 )
2872
2873 (define_insn "*neg_<shift><mode>3_compare0"
2874   [(set (reg:CC_NZ CC_REGNUM)
2875         (compare:CC_NZ
2876          (neg:GPI (ASHIFT:GPI
2877                    (match_operand:GPI 1 "register_operand" "r")
2878                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2879          (const_int 0)))
2880    (set (match_operand:GPI 0 "register_operand" "=r")
2881         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2882   ""
2883   "negs\\t%<w>0, %<w>1, <shift> %2"
2884   [(set_attr "type" "alus_shift_imm")]
2885 )
2886
2887 (define_insn "*neg_<shift>_<mode>2"
2888   [(set (match_operand:GPI 0 "register_operand" "=r")
2889         (neg:GPI (ASHIFT:GPI
2890                   (match_operand:GPI 1 "register_operand" "r")
2891                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2892   ""
2893   "neg\\t%<w>0, %<w>1, <shift> %2"
2894   [(set_attr "type" "alu_shift_imm")]
2895 )
2896
2897 ;; zero_extend version of above
2898 (define_insn "*neg_<shift>_si2_uxtw"
2899   [(set (match_operand:DI 0 "register_operand" "=r")
2900         (zero_extend:DI
2901          (neg:SI (ASHIFT:SI
2902                   (match_operand:SI 1 "register_operand" "r")
2903                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2904   ""
2905   "neg\\t%w0, %w1, <shift> %2"
2906   [(set_attr "type" "alu_shift_imm")]
2907 )
2908
2909 (define_insn "*neg_mul_imm_<mode>2"
2910   [(set (match_operand:GPI 0 "register_operand" "=r")
2911         (neg:GPI (mult:GPI
2912                   (match_operand:GPI 1 "register_operand" "r")
2913                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2914   ""
2915   "neg\\t%<w>0, %<w>1, lsl %p2"
2916   [(set_attr "type" "alu_shift_imm")]
2917 )
2918
2919 ;; zero_extend version of above
2920 (define_insn "*neg_mul_imm_si2_uxtw"
2921   [(set (match_operand:DI 0 "register_operand" "=r")
2922         (zero_extend:DI
2923          (neg:SI (mult:SI
2924                   (match_operand:SI 1 "register_operand" "r")
2925                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2926   ""
2927   "neg\\t%w0, %w1, lsl %p2"
2928   [(set_attr "type" "alu_shift_imm")]
2929 )
2930
2931 (define_insn "mul<mode>3"
2932   [(set (match_operand:GPI 0 "register_operand" "=r")
2933         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2934                   (match_operand:GPI 2 "register_operand" "r")))]
2935   ""
2936   "mul\\t%<w>0, %<w>1, %<w>2"
2937   [(set_attr "type" "mul")]
2938 )
2939
2940 ;; zero_extend version of above
2941 (define_insn "*mulsi3_uxtw"
2942   [(set (match_operand:DI 0 "register_operand" "=r")
2943         (zero_extend:DI
2944          (mult:SI (match_operand:SI 1 "register_operand" "r")
2945                   (match_operand:SI 2 "register_operand" "r"))))]
2946   ""
2947   "mul\\t%w0, %w1, %w2"
2948   [(set_attr "type" "mul")]
2949 )
2950
2951 (define_insn "madd<mode>"
2952   [(set (match_operand:GPI 0 "register_operand" "=r")
2953         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2954                             (match_operand:GPI 2 "register_operand" "r"))
2955                   (match_operand:GPI 3 "register_operand" "r")))]
2956   ""
2957   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2958   [(set_attr "type" "mla")]
2959 )
2960
2961 ;; zero_extend version of above
2962 (define_insn "*maddsi_uxtw"
2963   [(set (match_operand:DI 0 "register_operand" "=r")
2964         (zero_extend:DI
2965          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2966                            (match_operand:SI 2 "register_operand" "r"))
2967                   (match_operand:SI 3 "register_operand" "r"))))]
2968   ""
2969   "madd\\t%w0, %w1, %w2, %w3"
2970   [(set_attr "type" "mla")]
2971 )
2972
2973 (define_insn "*msub<mode>"
2974   [(set (match_operand:GPI 0 "register_operand" "=r")
2975         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2976                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2977                              (match_operand:GPI 2 "register_operand" "r"))))]
2978
2979   ""
2980   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2981   [(set_attr "type" "mla")]
2982 )
2983
2984 ;; zero_extend version of above
2985 (define_insn "*msubsi_uxtw"
2986   [(set (match_operand:DI 0 "register_operand" "=r")
2987         (zero_extend:DI
2988          (minus:SI (match_operand:SI 3 "register_operand" "r")
2989                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2990                             (match_operand:SI 2 "register_operand" "r")))))]
2991
2992   ""
2993   "msub\\t%w0, %w1, %w2, %w3"
2994   [(set_attr "type" "mla")]
2995 )
2996
2997 (define_insn "*mul<mode>_neg"
2998   [(set (match_operand:GPI 0 "register_operand" "=r")
2999         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3000                   (match_operand:GPI 2 "register_operand" "r")))]
3001
3002   ""
3003   "mneg\\t%<w>0, %<w>1, %<w>2"
3004   [(set_attr "type" "mul")]
3005 )
3006
3007 ;; zero_extend version of above
3008 (define_insn "*mulsi_neg_uxtw"
3009   [(set (match_operand:DI 0 "register_operand" "=r")
3010         (zero_extend:DI
3011          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3012                   (match_operand:SI 2 "register_operand" "r"))))]
3013
3014   ""
3015   "mneg\\t%w0, %w1, %w2"
3016   [(set_attr "type" "mul")]
3017 )
3018
3019 (define_insn "<su_optab>mulsidi3"
3020   [(set (match_operand:DI 0 "register_operand" "=r")
3021         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3022                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3023   ""
3024   "<su>mull\\t%0, %w1, %w2"
3025   [(set_attr "type" "<su>mull")]
3026 )
3027
3028 (define_insn "<su_optab>maddsidi4"
3029   [(set (match_operand:DI 0 "register_operand" "=r")
3030         (plus:DI (mult:DI
3031                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3032                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3033                  (match_operand:DI 3 "register_operand" "r")))]
3034   ""
3035   "<su>maddl\\t%0, %w1, %w2, %3"
3036   [(set_attr "type" "<su>mlal")]
3037 )
3038
3039 (define_insn "<su_optab>msubsidi4"
3040   [(set (match_operand:DI 0 "register_operand" "=r")
3041         (minus:DI
3042          (match_operand:DI 3 "register_operand" "r")
3043          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3044                   (ANY_EXTEND:DI
3045                    (match_operand:SI 2 "register_operand" "r")))))]
3046   ""
3047   "<su>msubl\\t%0, %w1, %w2, %3"
3048   [(set_attr "type" "<su>mlal")]
3049 )
3050
3051 (define_insn "*<su_optab>mulsidi_neg"
3052   [(set (match_operand:DI 0 "register_operand" "=r")
3053         (mult:DI (neg:DI
3054                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3055                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3056   ""
3057   "<su>mnegl\\t%0, %w1, %w2"
3058   [(set_attr "type" "<su>mull")]
3059 )
3060
3061 (define_expand "<su_optab>mulditi3"
3062   [(set (match_operand:TI 0 "register_operand")
3063         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3064                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3065   ""
3066 {
3067   rtx low = gen_reg_rtx (DImode);
3068   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3069
3070   rtx high = gen_reg_rtx (DImode);
3071   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3072
3073   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3074   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3075   DONE;
3076 })
3077
3078 ;; The default expansion of multi3 using umuldi3_highpart will perform
3079 ;; the additions in an order that fails to combine into two madd insns.
3080 (define_expand "multi3"
3081   [(set (match_operand:TI 0 "register_operand")
3082         (mult:TI (match_operand:TI 1 "register_operand")
3083                  (match_operand:TI 2 "register_operand")))]
3084   ""
3085 {
3086   rtx l0 = gen_reg_rtx (DImode);
3087   rtx l1 = gen_lowpart (DImode, operands[1]);
3088   rtx l2 = gen_lowpart (DImode, operands[2]);
3089   rtx h0 = gen_reg_rtx (DImode);
3090   rtx h1 = gen_highpart (DImode, operands[1]);
3091   rtx h2 = gen_highpart (DImode, operands[2]);
3092
3093   emit_insn (gen_muldi3 (l0, l1, l2));
3094   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3095   emit_insn (gen_madddi (h0, h1, l2, h0));
3096   emit_insn (gen_madddi (h0, l1, h2, h0));
3097
3098   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3099   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3100   DONE;
3101 })
3102
3103 (define_insn "<su>muldi3_highpart"
3104   [(set (match_operand:DI 0 "register_operand" "=r")
3105         (truncate:DI
3106          (lshiftrt:TI
3107           (mult:TI
3108            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3109            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3110           (const_int 64))))]
3111   ""
3112   "<su>mulh\\t%0, %1, %2"
3113   [(set_attr "type" "<su>mull")]
3114 )
3115
3116 (define_insn "<su_optab>div<mode>3"
3117   [(set (match_operand:GPI 0 "register_operand" "=r")
3118         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3119                      (match_operand:GPI 2 "register_operand" "r")))]
3120   ""
3121   "<su>div\\t%<w>0, %<w>1, %<w>2"
3122   [(set_attr "type" "<su>div")]
3123 )
3124
3125 ;; zero_extend version of above
3126 (define_insn "*<su_optab>divsi3_uxtw"
3127   [(set (match_operand:DI 0 "register_operand" "=r")
3128         (zero_extend:DI
3129          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3130                      (match_operand:SI 2 "register_operand" "r"))))]
3131   ""
3132   "<su>div\\t%w0, %w1, %w2"
3133   [(set_attr "type" "<su>div")]
3134 )
3135
3136 ;; -------------------------------------------------------------------
3137 ;; Comparison insns
3138 ;; -------------------------------------------------------------------
3139
3140 (define_insn "cmp<mode>"
3141   [(set (reg:CC CC_REGNUM)
3142         (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
3143                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3144   ""
3145   "@
3146    cmp\\t%<w>0, %<w>1
3147    cmp\\t%<w>0, %1
3148    cmn\\t%<w>0, #%n1"
3149   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3150 )
3151
3152 (define_insn "fcmp<mode>"
3153   [(set (reg:CCFP CC_REGNUM)
3154         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3155                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3156    "TARGET_FLOAT"
3157    "@
3158     fcmp\\t%<s>0, #0.0
3159     fcmp\\t%<s>0, %<s>1"
3160   [(set_attr "type" "fcmp<s>")]
3161 )
3162
3163 (define_insn "fcmpe<mode>"
3164   [(set (reg:CCFPE CC_REGNUM)
3165         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3166                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3167    "TARGET_FLOAT"
3168    "@
3169     fcmpe\\t%<s>0, #0.0
3170     fcmpe\\t%<s>0, %<s>1"
3171   [(set_attr "type" "fcmp<s>")]
3172 )
3173
3174 (define_insn "*cmp_swp_<shift>_reg<mode>"
3175   [(set (reg:CC_SWP CC_REGNUM)
3176         (compare:CC_SWP (ASHIFT:GPI
3177                          (match_operand:GPI 0 "register_operand" "r")
3178                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3179                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3180   ""
3181   "cmp\\t%<w>2, %<w>0, <shift> %1"
3182   [(set_attr "type" "alus_shift_imm")]
3183 )
3184
3185 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3186   [(set (reg:CC_SWP CC_REGNUM)
3187         (compare:CC_SWP (ANY_EXTEND:GPI
3188                          (match_operand:ALLX 0 "register_operand" "r"))
3189                         (match_operand:GPI 1 "register_operand" "r")))]
3190   ""
3191   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3192   [(set_attr "type" "alus_ext")]
3193 )
3194
3195 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3196   [(set (reg:CC_SWP CC_REGNUM)
3197         (compare:CC_SWP (ashift:GPI
3198                          (ANY_EXTEND:GPI
3199                           (match_operand:ALLX 0 "register_operand" "r"))
3200                          (match_operand 1 "aarch64_imm3" "Ui3"))
3201         (match_operand:GPI 2 "register_operand" "r")))]
3202   ""
3203   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3204   [(set_attr "type" "alus_ext")]
3205 )
3206
3207 ;; -------------------------------------------------------------------
3208 ;; Store-flag and conditional select insns
3209 ;; -------------------------------------------------------------------
3210
3211 (define_expand "cstore<mode>4"
3212   [(set (match_operand:SI 0 "register_operand" "")
3213         (match_operator:SI 1 "aarch64_comparison_operator"
3214          [(match_operand:GPI 2 "register_operand" "")
3215           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3216   ""
3217   "
3218   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3219                                       operands[3]);
3220   operands[3] = const0_rtx;
3221   "
3222 )
3223
3224 (define_expand "cstorecc4"
3225   [(set (match_operand:SI 0 "register_operand")
3226        (match_operator 1 "aarch64_comparison_operator_mode"
3227         [(match_operand 2 "cc_register")
3228          (match_operand 3 "const0_operand")]))]
3229   ""
3230 "{
3231   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3232   DONE;
3233 }")
3234
3235
3236 (define_expand "cstore<mode>4"
3237   [(set (match_operand:SI 0 "register_operand" "")
3238         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3239          [(match_operand:GPF 2 "register_operand" "")
3240           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3241   ""
3242   "
3243   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3244                                       operands[3]);
3245   operands[3] = const0_rtx;
3246   "
3247 )
3248
3249 (define_insn "aarch64_cstore<mode>"
3250   [(set (match_operand:ALLI 0 "register_operand" "=r")
3251         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3252          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3253   ""
3254   "cset\\t%<w>0, %m1"
3255   [(set_attr "type" "csel")]
3256 )
3257
3258 ;; For a 24-bit immediate CST we can optimize the compare for equality
3259 ;; and branch sequence from:
3260 ;;      mov     x0, #imm1
3261 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3262 ;;      cmp     x1, x0
3263 ;;      cset    x2, <ne,eq>
3264 ;; into the shorter:
3265 ;;      sub     x0, x1, #(CST & 0xfff000)
3266 ;;      subs    x0, x0, #(CST & 0x000fff)
3267 ;;      cset x2, <ne, eq>.
3268 (define_insn_and_split "*compare_cstore<mode>_insn"
3269   [(set (match_operand:GPI 0 "register_operand" "=r")
3270          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3271                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3272    (clobber (reg:CC CC_REGNUM))]
3273   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3274    && !aarch64_plus_operand (operands[2], <MODE>mode)
3275    && !reload_completed"
3276   "#"
3277   "&& true"
3278   [(const_int 0)]
3279   {
3280     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3281     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3282     rtx tmp = gen_reg_rtx (<MODE>mode);
3283     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3284     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3285     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3286     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3287     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3288     DONE;
3289   }
3290   [(set_attr "type" "csel")]
3291 )
3292
3293 ;; zero_extend version of the above
3294 (define_insn "*cstoresi_insn_uxtw"
3295   [(set (match_operand:DI 0 "register_operand" "=r")
3296         (zero_extend:DI
3297          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3298           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3299   ""
3300   "cset\\t%w0, %m1"
3301   [(set_attr "type" "csel")]
3302 )
3303
3304 (define_insn "cstore<mode>_neg"
3305   [(set (match_operand:ALLI 0 "register_operand" "=r")
3306         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3307                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
3308   ""
3309   "csetm\\t%<w>0, %m1"
3310   [(set_attr "type" "csel")]
3311 )
3312
3313 ;; zero_extend version of the above
3314 (define_insn "*cstoresi_neg_uxtw"
3315   [(set (match_operand:DI 0 "register_operand" "=r")
3316         (zero_extend:DI
3317          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3318                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3319   ""
3320   "csetm\\t%w0, %m1"
3321   [(set_attr "type" "csel")]
3322 )
3323
3324 (define_expand "cmov<mode>6"
3325   [(set (match_operand:GPI 0 "register_operand" "")
3326         (if_then_else:GPI
3327          (match_operator 1 "aarch64_comparison_operator"
3328           [(match_operand:GPI 2 "register_operand" "")
3329            (match_operand:GPI 3 "aarch64_plus_operand" "")])
3330          (match_operand:GPI 4 "register_operand" "")
3331          (match_operand:GPI 5 "register_operand" "")))]
3332   ""
3333   "
3334   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3335                                       operands[3]);
3336   operands[3] = const0_rtx;
3337   "
3338 )
3339
3340 (define_expand "cmov<mode>6"
3341   [(set (match_operand:GPF 0 "register_operand" "")
3342         (if_then_else:GPF
3343          (match_operator 1 "aarch64_comparison_operator"
3344           [(match_operand:GPF 2 "register_operand" "")
3345            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3346          (match_operand:GPF 4 "register_operand" "")
3347          (match_operand:GPF 5 "register_operand" "")))]
3348   ""
3349   "
3350   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3351                                       operands[3]);
3352   operands[3] = const0_rtx;
3353   "
3354 )
3355
3356 (define_insn "*cmov<mode>_insn"
3357   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3358         (if_then_else:ALLI
3359          (match_operator 1 "aarch64_comparison_operator"
3360           [(match_operand 2 "cc_register" "") (const_int 0)])
3361          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3362          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3363   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3364      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3365   ;; Final two alternatives should be unreachable, but included for completeness
3366   "@
3367    csel\\t%<w>0, %<w>3, %<w>4, %m1
3368    csinv\\t%<w>0, %<w>3, <w>zr, %m1
3369    csinv\\t%<w>0, %<w>4, <w>zr, %M1
3370    csinc\\t%<w>0, %<w>3, <w>zr, %m1
3371    csinc\\t%<w>0, %<w>4, <w>zr, %M1
3372    mov\\t%<w>0, -1
3373    mov\\t%<w>0, 1"
3374   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3375 )
3376
3377 ;; zero_extend version of above
3378 (define_insn "*cmovsi_insn_uxtw"
3379   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3380         (zero_extend:DI
3381          (if_then_else:SI
3382           (match_operator 1 "aarch64_comparison_operator"
3383            [(match_operand 2 "cc_register" "") (const_int 0)])
3384           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3385           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3386   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3387      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3388   ;; Final two alternatives should be unreachable, but included for completeness
3389   "@
3390    csel\\t%w0, %w3, %w4, %m1
3391    csinv\\t%w0, %w3, wzr, %m1
3392    csinv\\t%w0, %w4, wzr, %M1
3393    csinc\\t%w0, %w3, wzr, %m1
3394    csinc\\t%w0, %w4, wzr, %M1
3395    mov\\t%w0, -1
3396    mov\\t%w0, 1"
3397   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3398 )
3399
3400 (define_insn "*cmovdi_insn_uxtw"
3401   [(set (match_operand:DI 0 "register_operand" "=r")
3402         (if_then_else:DI
3403          (match_operator 1 "aarch64_comparison_operator"
3404           [(match_operand 2 "cc_register" "") (const_int 0)])
3405          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3406          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3407   ""
3408   "csel\\t%w0, %w3, %w4, %m1"
3409   [(set_attr "type" "csel")]
3410 )
3411
3412 (define_insn "*cmov<mode>_insn"
3413   [(set (match_operand:GPF 0 "register_operand" "=w")
3414         (if_then_else:GPF
3415          (match_operator 1 "aarch64_comparison_operator"
3416           [(match_operand 2 "cc_register" "") (const_int 0)])
3417          (match_operand:GPF 3 "register_operand" "w")
3418          (match_operand:GPF 4 "register_operand" "w")))]
3419   "TARGET_FLOAT"
3420   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3421   [(set_attr "type" "fcsel")]
3422 )
3423
3424 (define_expand "mov<mode>cc"
3425   [(set (match_operand:ALLI 0 "register_operand" "")
3426         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3427                            (match_operand:ALLI 2 "register_operand" "")
3428                            (match_operand:ALLI 3 "register_operand" "")))]
3429   ""
3430   {
3431     rtx ccreg;
3432     enum rtx_code code = GET_CODE (operands[1]);
3433
3434     if (code == UNEQ || code == LTGT)
3435       FAIL;
3436
3437     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3438                                      XEXP (operands[1], 1));
3439     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3440   }
3441 )
3442
3443 (define_expand "mov<GPF:mode><GPI:mode>cc"
3444   [(set (match_operand:GPI 0 "register_operand" "")
3445         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3446                           (match_operand:GPF 2 "register_operand" "")
3447                           (match_operand:GPF 3 "register_operand" "")))]
3448   ""
3449   {
3450     rtx ccreg;
3451     enum rtx_code code = GET_CODE (operands[1]);
3452
3453     if (code == UNEQ || code == LTGT)
3454       FAIL;
3455
3456     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3457                                   XEXP (operands[1], 1));
3458     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3459   }
3460 )
3461
3462 (define_expand "mov<mode>cc"
3463   [(set (match_operand:GPF 0 "register_operand" "")
3464         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3465                           (match_operand:GPF 2 "register_operand" "")
3466                           (match_operand:GPF 3 "register_operand" "")))]
3467   ""
3468   {
3469     rtx ccreg;
3470     enum rtx_code code = GET_CODE (operands[1]);
3471
3472     if (code == UNEQ || code == LTGT)
3473       FAIL;
3474
3475     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3476                                   XEXP (operands[1], 1));
3477     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3478   }
3479 )
3480
3481 (define_expand "<neg_not_op><mode>cc"
3482   [(set (match_operand:GPI 0 "register_operand" "")
3483         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3484                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3485                           (match_operand:GPI 3 "register_operand" "")))]
3486   ""
3487   {
3488     rtx ccreg;
3489     enum rtx_code code = GET_CODE (operands[1]);
3490
3491     if (code == UNEQ || code == LTGT)
3492       FAIL;
3493
3494     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3495                                       XEXP (operands[1], 1));
3496     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3497   }
3498 )
3499
3500 ;; CRC32 instructions.
3501 (define_insn "aarch64_<crc_variant>"
3502   [(set (match_operand:SI 0 "register_operand" "=r")
3503         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3504                     (match_operand:<crc_mode> 2 "register_operand" "r")]
3505          CRC))]
3506   "TARGET_CRC32"
3507   {
3508     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
3509       return "<crc_variant>\\t%w0, %w1, %x2";
3510     else
3511       return "<crc_variant>\\t%w0, %w1, %w2";
3512   }
3513   [(set_attr "type" "crc")]
3514 )
3515
3516 (define_insn "*csinc2<mode>_insn"
3517   [(set (match_operand:GPI 0 "register_operand" "=r")
3518         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3519                   (match_operand:GPI 1 "register_operand" "r")))]
3520   ""
3521   "cinc\\t%<w>0, %<w>1, %m2"
3522   [(set_attr "type" "csel")]
3523 )
3524
3525 (define_insn "csinc3<mode>_insn"
3526   [(set (match_operand:GPI 0 "register_operand" "=r")
3527         (if_then_else:GPI
3528           (match_operand 1 "aarch64_comparison_operation" "")
3529           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3530                     (const_int 1))
3531           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3532   ""
3533   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3534   [(set_attr "type" "csel")]
3535 )
3536
3537 (define_insn "*csinv3<mode>_insn"
3538   [(set (match_operand:GPI 0 "register_operand" "=r")
3539         (if_then_else:GPI
3540           (match_operand 1 "aarch64_comparison_operation" "")
3541           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3542           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3543   ""
3544   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3545   [(set_attr "type" "csel")]
3546 )
3547
3548 (define_insn "csneg3_uxtw_insn"
3549   [(set (match_operand:DI 0 "register_operand" "=r")
3550         (zero_extend:DI
3551           (if_then_else:SI
3552             (match_operand 1 "aarch64_comparison_operation" "")
3553             (neg:SI (match_operand:SI 2 "register_operand" "r"))
3554             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3555   ""
3556   "csneg\\t%w0, %w3, %w2, %M1"
3557   [(set_attr "type" "csel")]
3558 )
3559
3560 (define_insn "csneg3<mode>_insn"
3561   [(set (match_operand:GPI 0 "register_operand" "=r")
3562         (if_then_else:GPI
3563           (match_operand 1 "aarch64_comparison_operation" "")
3564           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3565           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3566   ""
3567   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3568   [(set_attr "type" "csel")]
3569 )
3570
3571 ;; If X can be loaded by a single CNT[BHWD] instruction,
3572 ;;
3573 ;;    A = UMAX (B, X)
3574 ;;
3575 ;; is equivalent to:
3576 ;;
3577 ;;    TMP = UQDEC[BHWD] (B, X)
3578 ;;    A = TMP + X
3579 ;;
3580 ;; Defining the pattern this way means that:
3581 ;;
3582 ;;    A = UMAX (B, X) - X
3583 ;;
3584 ;; becomes:
3585 ;;
3586 ;;    TMP1 = UQDEC[BHWD] (B, X)
3587 ;;    TMP2 = TMP1 + X
3588 ;;    A = TMP2 - X
3589 ;;
3590 ;; which combine can optimize to:
3591 ;;
3592 ;;    A = UQDEC[BHWD] (B, X)
3593 ;;
3594 ;; We don't use match_operand predicates because the order of the operands
3595 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
3596 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
3597 (define_expand "umax<mode>3"
3598   [(set (match_operand:GPI 0 "register_operand")
3599         (umax:GPI (match_operand:GPI 1 "")
3600                   (match_operand:GPI 2 "")))]
3601   "TARGET_SVE"
3602   {
3603     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
3604       std::swap (operands[1], operands[2]);
3605     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
3606       FAIL;
3607     rtx temp = gen_reg_rtx (<MODE>mode);
3608     operands[1] = force_reg (<MODE>mode, operands[1]);
3609     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
3610     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
3611     DONE;
3612   }
3613 )
3614
3615 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
3616 (define_insn "aarch64_uqdec<mode>"
3617   [(set (match_operand:GPI 0 "register_operand" "=r")
3618         (minus:GPI
3619          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
3620                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
3621          (match_dup 2)))]
3622   "TARGET_SVE"
3623   {
3624     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
3625   }
3626 )
3627
3628 ;; -------------------------------------------------------------------
3629 ;; Logical operations
3630 ;; -------------------------------------------------------------------
3631
3632
3633 (define_insn_and_split "*aarch64_and<mode>_imm2"
3634   [(set (match_operand:GPI 0 "register_operand" "=rk")
3635         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
3636                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
3637   ""
3638   "#"
3639   "true"
3640   [(const_int 0)]
3641   {
3642      HOST_WIDE_INT val = INTVAL (operands[2]);
3643      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
3644      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
3645
3646      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
3647      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
3648      DONE;
3649   }
3650 )
3651
3652 (define_insn "<optab><mode>3"
3653   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3654         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3655                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3656   ""
3657   "@
3658   <logical>\\t%<w>0, %<w>1, %<w>2
3659   <logical>\\t%<w>0, %<w>1, %2
3660   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3661   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3662    (set_attr "simd" "*,*,yes")]
3663 )
3664
3665 ;; zero_extend version of above
3666 (define_insn "*<optab>si3_uxtw"
3667   [(set (match_operand:DI 0 "register_operand" "=r,rk")
3668         (zero_extend:DI
3669          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3670                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3671   ""
3672   "@
3673    <logical>\\t%w0, %w1, %w2
3674    <logical>\\t%w0, %w1, %2"
3675   [(set_attr "type" "logic_reg,logic_imm")]
3676 )
3677
3678 (define_insn "*and<mode>3_compare0"
3679   [(set (reg:CC_NZ CC_REGNUM)
3680         (compare:CC_NZ
3681          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3682                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3683          (const_int 0)))
3684    (set (match_operand:GPI 0 "register_operand" "=r,r")
3685         (and:GPI (match_dup 1) (match_dup 2)))]
3686   ""
3687   "@
3688    ands\\t%<w>0, %<w>1, %<w>2
3689    ands\\t%<w>0, %<w>1, %2"
3690   [(set_attr "type" "logics_reg,logics_imm")]
3691 )
3692
3693 ;; zero_extend version of above
3694 (define_insn "*andsi3_compare0_uxtw"
3695   [(set (reg:CC_NZ CC_REGNUM)
3696         (compare:CC_NZ
3697          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3698                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3699          (const_int 0)))
3700    (set (match_operand:DI 0 "register_operand" "=r,r")
3701         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3702   ""
3703   "@
3704    ands\\t%w0, %w1, %w2
3705    ands\\t%w0, %w1, %2"
3706   [(set_attr "type" "logics_reg,logics_imm")]
3707 )
3708
3709 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3710   [(set (reg:CC_NZ CC_REGNUM)
3711         (compare:CC_NZ
3712          (and:GPI (SHIFT:GPI
3713                    (match_operand:GPI 1 "register_operand" "r")
3714                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3715                   (match_operand:GPI 3 "register_operand" "r"))
3716          (const_int 0)))
3717    (set (match_operand:GPI 0 "register_operand" "=r")
3718         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3719   ""
3720   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3721   [(set_attr "type" "logics_shift_imm")]
3722 )
3723
3724 ;; zero_extend version of above
3725 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3726   [(set (reg:CC_NZ CC_REGNUM)
3727         (compare:CC_NZ
3728          (and:SI (SHIFT:SI
3729                   (match_operand:SI 1 "register_operand" "r")
3730                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3731                  (match_operand:SI 3 "register_operand" "r"))
3732          (const_int 0)))
3733    (set (match_operand:DI 0 "register_operand" "=r")
3734         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3735                                 (match_dup 3))))]
3736   ""
3737   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3738   [(set_attr "type" "logics_shift_imm")]
3739 )
3740
3741 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3742   [(set (match_operand:GPI 0 "register_operand" "=r")
3743         (LOGICAL:GPI (SHIFT:GPI
3744                       (match_operand:GPI 1 "register_operand" "r")
3745                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3746                      (match_operand:GPI 3 "register_operand" "r")))]
3747   ""
3748   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3749   [(set_attr "type" "logic_shift_imm")]
3750 )
3751
3752 (define_insn "*<optab>_rol<mode>3"
3753   [(set (match_operand:GPI 0 "register_operand" "=r")
3754         (LOGICAL:GPI (rotate:GPI
3755                       (match_operand:GPI 1 "register_operand" "r")
3756                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3757                      (match_operand:GPI 3 "register_operand" "r")))]
3758   ""
3759   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3760   [(set_attr "type" "logic_shift_imm")]
3761 )
3762
3763 ;; zero_extend versions of above
3764 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3765   [(set (match_operand:DI 0 "register_operand" "=r")
3766         (zero_extend:DI
3767          (LOGICAL:SI (SHIFT:SI
3768                       (match_operand:SI 1 "register_operand" "r")
3769                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3770                      (match_operand:SI 3 "register_operand" "r"))))]
3771   ""
3772   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3773   [(set_attr "type" "logic_shift_imm")]
3774 )
3775
3776 (define_insn "*<optab>_rolsi3_uxtw"
3777   [(set (match_operand:DI 0 "register_operand" "=r")
3778         (zero_extend:DI
3779          (LOGICAL:SI (rotate:SI
3780                       (match_operand:SI 1 "register_operand" "r")
3781                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3782                      (match_operand:SI 3 "register_operand" "r"))))]
3783   ""
3784   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3785   [(set_attr "type" "logic_shift_imm")]
3786 )
3787
3788 (define_insn "one_cmpl<mode>2"
3789   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3790         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3791   ""
3792   "@
3793   mvn\\t%<w>0, %<w>1
3794   mvn\\t%0.8b, %1.8b"
3795   [(set_attr "type" "logic_reg,neon_logic")
3796    (set_attr "simd" "*,yes")]
3797 )
3798
3799 (define_insn "*one_cmpl_<optab><mode>2"
3800   [(set (match_operand:GPI 0 "register_operand" "=r")
3801         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3802                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3803   ""
3804   "mvn\\t%<w>0, %<w>1, <shift> %2"
3805   [(set_attr "type" "logic_shift_imm")]
3806 )
3807
3808 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3809
3810 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3811   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3812         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3813                      (match_operand:GPI 2 "register_operand" "r,w")))]
3814   ""
3815   "@
3816   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3817   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3818   [(set_attr "type" "logic_reg,neon_logic")
3819    (set_attr "simd" "*,yes")]
3820 )
3821
3822 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3823   [(set (match_operand:DI 0 "register_operand" "=r")
3824         (zero_extend:DI
3825           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3826                        (match_operand:SI 2 "register_operand" "r"))))]
3827   ""
3828   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3829   [(set_attr "type" "logic_reg")]
3830 )
3831
3832 (define_insn "*xor_one_cmplsidi3_ze"
3833   [(set (match_operand:DI 0 "register_operand" "=r")
3834         (zero_extend:DI
3835           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3836                           (match_operand:SI 2 "register_operand" "r")))))]
3837   ""
3838   "eon\\t%w0, %w1, %w2"
3839   [(set_attr "type" "logic_reg")]
3840 )
3841
3842 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3843 ;; eon does not operate on SIMD registers so the vector variant must be split.
3844 (define_insn_and_split "*xor_one_cmpl<mode>3"
3845   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3846         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3847                           (match_operand:GPI 2 "register_operand" "r,w"))))]
3848   ""
3849   "@
3850   eon\\t%<w>0, %<w>1, %<w>2
3851   #"
3852   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3853   [(set (match_operand:GPI 0 "register_operand" "=w")
3854         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3855                  (match_operand:GPI 2 "register_operand" "w")))
3856    (set (match_dup 0) (not:GPI (match_dup 0)))]
3857   ""
3858   [(set_attr "type" "logic_reg,multiple")
3859    (set_attr "simd" "*,yes")]
3860 )
3861
3862 (define_insn "*and_one_cmpl<mode>3_compare0"
3863   [(set (reg:CC_NZ CC_REGNUM)
3864         (compare:CC_NZ
3865          (and:GPI (not:GPI
3866                    (match_operand:GPI 1 "register_operand" "r"))
3867                   (match_operand:GPI 2 "register_operand" "r"))
3868          (const_int 0)))
3869    (set (match_operand:GPI 0 "register_operand" "=r")
3870         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3871   ""
3872   "bics\\t%<w>0, %<w>2, %<w>1"
3873   [(set_attr "type" "logics_reg")]
3874 )
3875
3876 ;; zero_extend version of above
3877 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3878   [(set (reg:CC_NZ CC_REGNUM)
3879         (compare:CC_NZ
3880          (and:SI (not:SI
3881                   (match_operand:SI 1 "register_operand" "r"))
3882                  (match_operand:SI 2 "register_operand" "r"))
3883          (const_int 0)))
3884    (set (match_operand:DI 0 "register_operand" "=r")
3885         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3886   ""
3887   "bics\\t%w0, %w2, %w1"
3888   [(set_attr "type" "logics_reg")]
3889 )
3890
3891 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3892   [(set (reg:CC_NZ CC_REGNUM)
3893     (compare:CC_NZ
3894      (and:GPI (not:GPI
3895            (match_operand:GPI 0 "register_operand" "r"))
3896           (match_operand:GPI 1 "register_operand" "r"))
3897      (const_int 0)))]
3898   ""
3899   "bics\\t<w>zr, %<w>1, %<w>0"
3900   [(set_attr "type" "logics_reg")]
3901 )
3902
3903 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3904   [(set (match_operand:GPI 0 "register_operand" "=r")
3905         (LOGICAL:GPI (not:GPI
3906                       (SHIFT:GPI
3907                        (match_operand:GPI 1 "register_operand" "r")
3908                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3909                      (match_operand:GPI 3 "register_operand" "r")))]
3910   ""
3911   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3912   [(set_attr "type" "logic_shift_imm")]
3913 )
3914
3915 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3916   [(set (match_operand:GPI 0 "register_operand" "=r")
3917         (not:GPI (xor:GPI
3918                       (SHIFT:GPI
3919                        (match_operand:GPI 1 "register_operand" "r")
3920                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3921                      (match_operand:GPI 3 "register_operand" "r"))))]
3922   ""
3923   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3924   [(set_attr "type" "logic_shift_imm")]
3925 )
3926
3927 ;; Zero-extend version of the above.
3928 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3929   [(set (match_operand:DI 0 "register_operand" "=r")
3930         (zero_extend:DI
3931           (not:SI (xor:SI
3932                     (SHIFT:SI
3933                       (match_operand:SI 1 "register_operand" "r")
3934                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3935                     (match_operand:SI 3 "register_operand" "r")))))]
3936   ""
3937   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3938   [(set_attr "type" "logic_shift_imm")]
3939 )
3940
3941 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3942   [(set (reg:CC_NZ CC_REGNUM)
3943         (compare:CC_NZ
3944          (and:GPI (not:GPI
3945                    (SHIFT:GPI
3946                     (match_operand:GPI 1 "register_operand" "r")
3947                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3948                   (match_operand:GPI 3 "register_operand" "r"))
3949          (const_int 0)))
3950    (set (match_operand:GPI 0 "register_operand" "=r")
3951         (and:GPI (not:GPI
3952                   (SHIFT:GPI
3953                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
3954   ""
3955   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3956   [(set_attr "type" "logics_shift_imm")]
3957 )
3958
3959 ;; zero_extend version of above
3960 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3961   [(set (reg:CC_NZ CC_REGNUM)
3962         (compare:CC_NZ
3963          (and:SI (not:SI
3964                   (SHIFT:SI
3965                    (match_operand:SI 1 "register_operand" "r")
3966                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3967                  (match_operand:SI 3 "register_operand" "r"))
3968          (const_int 0)))
3969    (set (match_operand:DI 0 "register_operand" "=r")
3970         (zero_extend:DI (and:SI
3971                          (not:SI
3972                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3973   ""
3974   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3975   [(set_attr "type" "logics_shift_imm")]
3976 )
3977
3978 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3979   [(set (reg:CC_NZ CC_REGNUM)
3980     (compare:CC_NZ
3981      (and:GPI (not:GPI
3982            (SHIFT:GPI
3983             (match_operand:GPI 0 "register_operand" "r")
3984             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3985           (match_operand:GPI 2 "register_operand" "r"))
3986      (const_int 0)))]
3987   ""
3988   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3989   [(set_attr "type" "logics_shift_imm")]
3990 )
3991
3992 (define_insn "clz<mode>2"
3993   [(set (match_operand:GPI 0 "register_operand" "=r")
3994         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3995   ""
3996   "clz\\t%<w>0, %<w>1"
3997   [(set_attr "type" "clz")]
3998 )
3999
4000 (define_expand "ffs<mode>2"
4001   [(match_operand:GPI 0 "register_operand")
4002    (match_operand:GPI 1 "register_operand")]
4003   ""
4004   {
4005     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4006     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4007
4008     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4009     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4010     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4011     DONE;
4012   }
4013 )
4014
4015 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4016 ;;
4017 ;; MOV  v.1d, x0
4018 ;; CNT  v1.8b, v.8b
4019 ;; ADDV b2, v1.8b
4020 ;; MOV  w0, v2.b[0]
4021
4022 (define_expand "popcount<mode>2"
4023   [(match_operand:GPI 0 "register_operand")
4024    (match_operand:GPI 1 "register_operand")]
4025   "TARGET_SIMD"
4026 {
4027   rtx v = gen_reg_rtx (V8QImode);
4028   rtx v1 = gen_reg_rtx (V8QImode);
4029   rtx r = gen_reg_rtx (QImode);
4030   rtx in = operands[1];
4031   rtx out = operands[0];
4032   if(<MODE>mode == SImode)
4033     {
4034       rtx tmp;
4035       tmp = gen_reg_rtx (DImode);
4036       /* If we have SImode, zero extend to DImode, pop count does
4037          not change if we have extra zeros. */
4038       emit_insn (gen_zero_extendsidi2 (tmp, in));
4039       in = tmp;
4040     }
4041   emit_move_insn (v, gen_lowpart (V8QImode, in));
4042   emit_insn (gen_popcountv8qi2 (v1, v));
4043   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4044   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4045   DONE;
4046 })
4047
4048 (define_insn "clrsb<mode>2"
4049   [(set (match_operand:GPI 0 "register_operand" "=r")
4050         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4051   ""
4052   "cls\\t%<w>0, %<w>1"
4053   [(set_attr "type" "clz")]
4054 )
4055
4056 (define_insn "rbit<mode>2"
4057   [(set (match_operand:GPI 0 "register_operand" "=r")
4058         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4059   ""
4060   "rbit\\t%<w>0, %<w>1"
4061   [(set_attr "type" "rbit")]
4062 )
4063
4064 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4065 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4066 ;; expression and split after reload to enable scheduling them apart if
4067 ;; needed.
4068
4069 (define_insn_and_split "ctz<mode>2"
4070  [(set (match_operand:GPI           0 "register_operand" "=r")
4071        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4072   ""
4073   "#"
4074   "reload_completed"
4075   [(const_int 0)]
4076   "
4077   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4078   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4079   DONE;
4080 ")
4081
4082 (define_insn "*and<mode>_compare0"
4083   [(set (reg:CC_NZ CC_REGNUM)
4084         (compare:CC_NZ
4085          (match_operand:SHORT 0 "register_operand" "r")
4086          (const_int 0)))]
4087   ""
4088   "tst\\t%<w>0, <short_mask>"
4089   [(set_attr "type" "alus_imm")]
4090 )
4091
4092 (define_insn "*ands<mode>_compare0"
4093   [(set (reg:CC_NZ CC_REGNUM)
4094         (compare:CC_NZ
4095          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4096          (const_int 0)))
4097    (set (match_operand:GPI 0 "register_operand" "=r")
4098         (zero_extend:GPI (match_dup 1)))]
4099   ""
4100   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4101   [(set_attr "type" "alus_imm")]
4102 )
4103
4104 (define_insn "*and<mode>3nr_compare0"
4105   [(set (reg:CC_NZ CC_REGNUM)
4106         (compare:CC_NZ
4107          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4108                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4109          (const_int 0)))]
4110   ""
4111   "@
4112    tst\\t%<w>0, %<w>1
4113    tst\\t%<w>0, %1"
4114   [(set_attr "type" "logics_reg,logics_imm")]
4115 )
4116
4117 (define_split
4118   [(set (reg:CC_NZ CC_REGNUM)
4119         (compare:CC_NZ
4120          (and:GPI (match_operand:GPI 0 "register_operand")
4121                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4122          (const_int 0)))
4123    (clobber (match_operand:SI 2 "register_operand"))]
4124   ""
4125   [(set (match_dup 2) (match_dup 1))
4126    (set (reg:CC_NZ CC_REGNUM)
4127         (compare:CC_NZ
4128          (and:GPI (match_dup 0)
4129                   (match_dup 2))
4130          (const_int 0)))]
4131 )
4132
4133 (define_insn "*and<mode>3nr_compare0_zextract"
4134   [(set (reg:CC_NZ CC_REGNUM)
4135         (compare:CC_NZ
4136          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4137                   (match_operand:GPI 1 "const_int_operand" "n")
4138                   (match_operand:GPI 2 "const_int_operand" "n"))
4139          (const_int 0)))]
4140   "INTVAL (operands[1]) > 0
4141    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4142         <= GET_MODE_BITSIZE (<MODE>mode))
4143    && aarch64_bitmask_imm (
4144         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4145                                                  operands[2])),
4146         <MODE>mode)"
4147   {
4148     operands[1]
4149       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4150     return "tst\\t%<w>0, %1";
4151   }
4152   [(set_attr "type" "logics_shift_imm")]
4153 )
4154
4155 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4156   [(set (reg:CC_NZ CC_REGNUM)
4157         (compare:CC_NZ
4158          (and:GPI (SHIFT:GPI
4159                    (match_operand:GPI 0 "register_operand" "r")
4160                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4161                   (match_operand:GPI 2 "register_operand" "r"))
4162         (const_int 0)))]
4163   ""
4164   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4165   [(set_attr "type" "logics_shift_imm")]
4166 )
4167
4168 (define_split
4169   [(set (reg:CC_NZ CC_REGNUM)
4170         (compare:CC_NZ
4171          (and:GPI (SHIFT:GPI
4172                    (match_operand:GPI 0 "register_operand")
4173                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4174                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4175         (const_int 0)))
4176     (clobber (match_operand:SI 3 "register_operand"))]
4177   ""
4178   [(set (match_dup 3) (match_dup 2))
4179    (set (reg:CC_NZ CC_REGNUM)
4180         (compare:CC_NZ
4181          (and:GPI (SHIFT:GPI
4182                    (match_dup 0)
4183                    (match_dup 1))
4184                   (match_dup 3))
4185          (const_int 0)))]
4186 )
4187
4188 ;; -------------------------------------------------------------------
4189 ;; Shifts
4190 ;; -------------------------------------------------------------------
4191
4192 (define_expand "<optab><mode>3"
4193   [(set (match_operand:GPI 0 "register_operand")
4194         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4195                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4196   ""
4197   {
4198     if (CONST_INT_P (operands[2]))
4199       {
4200         operands[2] = GEN_INT (INTVAL (operands[2])
4201                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4202
4203         if (operands[2] == const0_rtx)
4204           {
4205             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4206             DONE;
4207           }
4208       }
4209   }
4210 )
4211
4212 (define_expand "ashl<mode>3"
4213   [(set (match_operand:SHORT 0 "register_operand")
4214         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4215                       (match_operand:QI 2 "const_int_operand")))]
4216   ""
4217   {
4218     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4219
4220     if (operands[2] == const0_rtx)
4221       {
4222         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4223         DONE;
4224       }
4225   }
4226 )
4227
4228 (define_expand "rotr<mode>3"
4229   [(set (match_operand:GPI 0 "register_operand")
4230         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4231                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4232   ""
4233   {
4234     if (CONST_INT_P (operands[2]))
4235       {
4236         operands[2] = GEN_INT (INTVAL (operands[2])
4237                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4238
4239         if (operands[2] == const0_rtx)
4240           {
4241             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4242             DONE;
4243           }
4244       }
4245   }
4246 )
4247
4248 (define_expand "rotl<mode>3"
4249   [(set (match_operand:GPI 0 "register_operand")
4250         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4251                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4252   ""
4253   {
4254     /* (SZ - cnt) % SZ == -cnt % SZ */
4255     if (CONST_INT_P (operands[2]))
4256       {
4257         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4258                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4259         if (operands[2] == const0_rtx)
4260           {
4261             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4262             DONE;
4263           }
4264       }
4265     else
4266       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4267                                         NULL_RTX, 1);
4268   }
4269 )
4270
4271 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4272 ;; they truncate the shift/rotate amount by the size of the registers they
4273 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
4274 ;; such redundant masking instructions.  GCC can do that automatically when
4275 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4276 ;; because some of the SISD shift alternatives don't perform this truncations.
4277 ;; So this pattern exists to catch such cases.
4278
4279 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4280   [(set (match_operand:GPI 0 "register_operand" "=r")
4281         (SHIFT:GPI
4282           (match_operand:GPI 1 "register_operand" "r")
4283           (match_operator 4 "subreg_lowpart_operator"
4284            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4285                      (match_operand 3 "const_int_operand" "n"))])))]
4286   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4287   "<shift>\t%<w>0, %<w>1, %<w>2"
4288   [(set_attr "type" "shift_reg")]
4289 )
4290
4291 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4292   [(set (match_operand:GPI 0 "register_operand" "=&r")
4293         (SHIFT:GPI
4294           (match_operand:GPI 1 "register_operand" "r")
4295           (match_operator 4 "subreg_lowpart_operator"
4296           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4297                            (match_operand 3 "const_int_operand" "n")))])))]
4298   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4299   "#"
4300   "&& true"
4301   [(const_int 0)]
4302   {
4303     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4304                : lowpart_subreg (SImode, operands[0], <MODE>mode));
4305     emit_insn (gen_negsi2 (tmp, operands[2]));
4306
4307     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4308     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4309                                      SUBREG_BYTE (operands[4]));
4310     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4311     DONE;
4312   }
4313 )
4314
4315 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
4316   [(set (match_operand:GPI 0 "register_operand" "=&r")
4317         (ashift:GPI
4318           (match_operand:GPI 1 "register_operand" "r")
4319           (minus:QI (match_operand 2 "const_int_operand" "n")
4320                     (match_operator 5 "subreg_lowpart_operator"
4321                     [(and:SI (match_operand:SI 3 "register_operand" "r")
4322                              (match_operand 4 "const_int_operand" "n"))]))))]
4323   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4324    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4325   "#"
4326   "&& true"
4327   [(const_int 0)]
4328   {
4329     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4330                : operands[0]);
4331
4332     emit_insn (gen_negsi2 (tmp, operands[3]));
4333
4334     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
4335     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
4336                                      SUBREG_BYTE (operands[5]));
4337
4338     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
4339     DONE;
4340   }
4341 )
4342
4343 (define_insn "*aarch64_<optab>_reg_di3_mask2"
4344   [(set (match_operand:DI 0 "register_operand" "=r")
4345         (SHIFT:DI
4346           (match_operand:DI 1 "register_operand" "r")
4347           (match_operator 4 "subreg_lowpart_operator"
4348            [(and:SI (match_operand:SI 2 "register_operand" "r")
4349                     (match_operand 3 "const_int_operand" "n"))])))]
4350   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
4351 {
4352   rtx xop[3];
4353   xop[0] = operands[0];
4354   xop[1] = operands[1];
4355   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
4356   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
4357   return "";
4358 }
4359   [(set_attr "type" "shift_reg")]
4360 )
4361
4362 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
4363   [(set (match_operand:GPI 0 "register_operand" "=&r")
4364         (ASHIFT:GPI
4365           (match_operand:GPI 1 "register_operand" "r")
4366           (minus:QI (match_operand 2 "const_int_operand" "n")
4367                     (match_operand:QI 3 "register_operand" "r"))))]
4368   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4369   "#"
4370   "&& true"
4371   [(const_int 0)]
4372   {
4373     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
4374
4375     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4376                : gen_lowpart (SImode, operands[0]));
4377
4378     emit_insn (gen_negsi2 (tmp, subreg_tmp));
4379
4380     rtx and_op = gen_rtx_AND (SImode, tmp,
4381                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
4382
4383     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
4384
4385     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
4386     DONE;
4387   }
4388   [(set_attr "length" "8")]
4389 )
4390
4391 ;; Logical left shift using SISD or Integer instruction
4392 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
4393   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
4394         (ashift:GPI
4395           (match_operand:GPI 1 "register_operand" "r,r,w,w")
4396           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
4397   ""
4398   "@
4399    lsl\t%<w>0, %<w>1, %2
4400    lsl\t%<w>0, %<w>1, %<w>2
4401    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4402    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
4403   [(set_attr "simd" "no,no,yes,yes")
4404    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
4405 )
4406
4407 ;; Logical right shift using SISD or Integer instruction
4408 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
4409   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4410         (lshiftrt:GPI
4411          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4412          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
4413                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4414   ""
4415   "@
4416    lsr\t%<w>0, %<w>1, %2
4417    lsr\t%<w>0, %<w>1, %<w>2
4418    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4419    #
4420    #"
4421   [(set_attr "simd" "no,no,yes,yes,yes")
4422    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4423 )
4424
4425 (define_split
4426   [(set (match_operand:DI 0 "aarch64_simd_register")
4427         (lshiftrt:DI
4428            (match_operand:DI 1 "aarch64_simd_register")
4429            (match_operand:QI 2 "aarch64_simd_register")))]
4430   "TARGET_SIMD && reload_completed"
4431   [(set (match_dup 3)
4432         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4433    (set (match_dup 0)
4434         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4435   {
4436     operands[3] = gen_lowpart (QImode, operands[0]);
4437   }
4438 )
4439
4440 (define_split
4441   [(set (match_operand:SI 0 "aarch64_simd_register")
4442         (lshiftrt:SI
4443            (match_operand:SI 1 "aarch64_simd_register")
4444            (match_operand:QI 2 "aarch64_simd_register")))]
4445   "TARGET_SIMD && reload_completed"
4446   [(set (match_dup 3)
4447         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4448    (set (match_dup 0)
4449         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4450   {
4451     operands[3] = gen_lowpart (QImode, operands[0]);
4452   }
4453 )
4454
4455 ;; Arithmetic right shift using SISD or Integer instruction
4456 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4457   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4458         (ashiftrt:GPI
4459           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4460           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
4461                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4462   ""
4463   "@
4464    asr\t%<w>0, %<w>1, %2
4465    asr\t%<w>0, %<w>1, %<w>2
4466    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4467    #
4468    #"
4469   [(set_attr "simd" "no,no,yes,yes,yes")
4470    (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4471 )
4472
4473 (define_split
4474   [(set (match_operand:DI 0 "aarch64_simd_register")
4475         (ashiftrt:DI
4476            (match_operand:DI 1 "aarch64_simd_register")
4477            (match_operand:QI 2 "aarch64_simd_register")))]
4478   "TARGET_SIMD && reload_completed"
4479   [(set (match_dup 3)
4480         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4481    (set (match_dup 0)
4482         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4483 {
4484   operands[3] = gen_lowpart (QImode, operands[0]);
4485 }
4486 )
4487
4488 (define_split
4489   [(set (match_operand:SI 0 "aarch64_simd_register")
4490         (ashiftrt:SI
4491            (match_operand:SI 1 "aarch64_simd_register")
4492            (match_operand:QI 2 "aarch64_simd_register")))]
4493   "TARGET_SIMD && reload_completed"
4494   [(set (match_dup 3)
4495         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4496    (set (match_dup 0)
4497         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4498 {
4499   operands[3] = gen_lowpart (QImode, operands[0]);
4500 }
4501 )
4502
4503 (define_insn "*aarch64_sisd_ushl"
4504   [(set (match_operand:DI 0 "register_operand" "=w")
4505         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4506                     (match_operand:QI 2 "register_operand" "w")]
4507                    UNSPEC_SISD_USHL))]
4508   "TARGET_SIMD"
4509   "ushl\t%d0, %d1, %d2"
4510   [(set_attr "simd" "yes")
4511    (set_attr "type" "neon_shift_reg")]
4512 )
4513
4514 (define_insn "*aarch64_ushl_2s"
4515   [(set (match_operand:SI 0 "register_operand" "=w")
4516         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4517                     (match_operand:QI 2 "register_operand" "w")]
4518                    UNSPEC_USHL_2S))]
4519   "TARGET_SIMD"
4520   "ushl\t%0.2s, %1.2s, %2.2s"
4521   [(set_attr "simd" "yes")
4522    (set_attr "type" "neon_shift_reg")]
4523 )
4524
4525 (define_insn "*aarch64_sisd_sshl"
4526   [(set (match_operand:DI 0 "register_operand" "=w")
4527         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4528                     (match_operand:QI 2 "register_operand" "w")]
4529                    UNSPEC_SISD_SSHL))]
4530   "TARGET_SIMD"
4531   "sshl\t%d0, %d1, %d2"
4532   [(set_attr "simd" "yes")
4533    (set_attr "type" "neon_shift_reg")]
4534 )
4535
4536 (define_insn "*aarch64_sshl_2s"
4537   [(set (match_operand:SI 0 "register_operand" "=w")
4538         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4539                     (match_operand:QI 2 "register_operand" "w")]
4540                    UNSPEC_SSHL_2S))]
4541   "TARGET_SIMD"
4542   "sshl\t%0.2s, %1.2s, %2.2s"
4543   [(set_attr "simd" "yes")
4544    (set_attr "type" "neon_shift_reg")]
4545 )
4546
4547 (define_insn "*aarch64_sisd_neg_qi"
4548   [(set (match_operand:QI 0 "register_operand" "=w")
4549         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4550                    UNSPEC_SISD_NEG))]
4551   "TARGET_SIMD"
4552   "neg\t%d0, %d1"
4553   [(set_attr "simd" "yes")
4554    (set_attr "type" "neon_neg")]
4555 )
4556
4557 ;; Rotate right
4558 (define_insn "*ror<mode>3_insn"
4559   [(set (match_operand:GPI 0 "register_operand" "=r,r")
4560      (rotatert:GPI
4561        (match_operand:GPI 1 "register_operand" "r,r")
4562        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
4563   ""
4564   "@
4565    ror\\t%<w>0, %<w>1, %2
4566    ror\\t%<w>0, %<w>1, %<w>2"
4567   [(set_attr "type" "rotate_imm,shift_reg")]
4568 )
4569
4570 ;; zero_extend version of above
4571 (define_insn "*<optab>si3_insn_uxtw"
4572   [(set (match_operand:DI 0 "register_operand" "=r,r")
4573         (zero_extend:DI (SHIFT:SI
4574          (match_operand:SI 1 "register_operand" "r,r")
4575          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
4576   ""
4577   "@
4578    <shift>\\t%w0, %w1, %2
4579    <shift>\\t%w0, %w1, %w2"
4580   [(set_attr "type" "bfx,shift_reg")]
4581 )
4582
4583 (define_insn "*<optab><mode>3_insn"
4584   [(set (match_operand:SHORT 0 "register_operand" "=r")
4585         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
4586                       (match_operand 2 "const_int_operand" "n")))]
4587   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4588 {
4589   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4590   return "<bfshift>\t%w0, %w1, %2, %3";
4591 }
4592   [(set_attr "type" "bfx")]
4593 )
4594
4595 (define_insn "*extr<mode>5_insn"
4596   [(set (match_operand:GPI 0 "register_operand" "=r")
4597         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4598                              (match_operand 3 "const_int_operand" "n"))
4599                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4600                                (match_operand 4 "const_int_operand" "n"))))]
4601   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
4602    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
4603   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4604   [(set_attr "type" "rotate_imm")]
4605 )
4606
4607 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
4608 ;; so we have to match both orderings.
4609 (define_insn "*extr<mode>5_insn_alt"
4610   [(set (match_operand:GPI 0 "register_operand" "=r")
4611         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4612                                 (match_operand 4 "const_int_operand" "n"))
4613                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4614                               (match_operand 3 "const_int_operand" "n"))))]
4615   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
4616    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
4617        == GET_MODE_BITSIZE (<MODE>mode))"
4618   "extr\\t%<w>0, %<w>1, %<w>2, %4"
4619   [(set_attr "type" "rotate_imm")]
4620 )
4621
4622 ;; zero_extend version of the above
4623 (define_insn "*extrsi5_insn_uxtw"
4624   [(set (match_operand:DI 0 "register_operand" "=r")
4625         (zero_extend:DI
4626          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
4627                             (match_operand 3 "const_int_operand" "n"))
4628                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4629                               (match_operand 4 "const_int_operand" "n")))))]
4630   "UINTVAL (operands[3]) < 32 &&
4631    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4632   "extr\\t%w0, %w1, %w2, %4"
4633   [(set_attr "type" "rotate_imm")]
4634 )
4635
4636 (define_insn "*extrsi5_insn_uxtw_alt"
4637   [(set (match_operand:DI 0 "register_operand" "=r")
4638         (zero_extend:DI
4639          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4640                                (match_operand 4 "const_int_operand" "n"))
4641                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
4642                             (match_operand 3 "const_int_operand" "n")))))]
4643   "UINTVAL (operands[3]) < 32 &&
4644    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4645   "extr\\t%w0, %w1, %w2, %4"
4646   [(set_attr "type" "rotate_imm")]
4647 )
4648
4649 (define_insn "*ror<mode>3_insn"
4650   [(set (match_operand:GPI 0 "register_operand" "=r")
4651         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
4652                     (match_operand 2 "const_int_operand" "n")))]
4653   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4654 {
4655   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4656   return "ror\\t%<w>0, %<w>1, %3";
4657 }
4658   [(set_attr "type" "rotate_imm")]
4659 )
4660
4661 ;; zero_extend version of the above
4662 (define_insn "*rorsi3_insn_uxtw"
4663   [(set (match_operand:DI 0 "register_operand" "=r")
4664         (zero_extend:DI
4665          (rotate:SI (match_operand:SI 1 "register_operand" "r")
4666                     (match_operand 2 "const_int_operand" "n"))))]
4667   "UINTVAL (operands[2]) < 32"
4668 {
4669   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
4670   return "ror\\t%w0, %w1, %3";
4671 }
4672   [(set_attr "type" "rotate_imm")]
4673 )
4674
4675 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
4676   [(set (match_operand:GPI 0 "register_operand" "=r")
4677         (ANY_EXTEND:GPI
4678          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
4679                        (match_operand 2 "const_int_operand" "n"))))]
4680   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4681 {
4682   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4683   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4684 }
4685   [(set_attr "type" "bfx")]
4686 )
4687
4688 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
4689   [(set (match_operand:GPI 0 "register_operand" "=r")
4690         (zero_extend:GPI
4691          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4692                          (match_operand 2 "const_int_operand" "n"))))]
4693   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4694 {
4695   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4696   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4697 }
4698   [(set_attr "type" "bfx")]
4699 )
4700
4701 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
4702   [(set (match_operand:GPI 0 "register_operand" "=r")
4703         (sign_extend:GPI
4704          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4705                          (match_operand 2 "const_int_operand" "n"))))]
4706   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4707 {
4708   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4709   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4710 }
4711   [(set_attr "type" "bfx")]
4712 )
4713
4714 ;; -------------------------------------------------------------------
4715 ;; Bitfields
4716 ;; -------------------------------------------------------------------
4717
4718 (define_expand "<optab>"
4719   [(set (match_operand:DI 0 "register_operand" "=r")
4720         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
4721                         (match_operand 2
4722                           "aarch64_simd_shift_imm_offset_di")
4723                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
4724   ""
4725   {
4726     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4727                    1, GET_MODE_BITSIZE (DImode) - 1))
4728      FAIL;
4729   }
4730 )
4731
4732
4733 (define_insn "*<optab><mode>"
4734   [(set (match_operand:GPI 0 "register_operand" "=r")
4735         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
4736                          (match_operand 2
4737                            "aarch64_simd_shift_imm_offset_<mode>" "n")
4738                          (match_operand 3
4739                            "aarch64_simd_shift_imm_<mode>" "n")))]
4740   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4741              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
4742   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
4743   [(set_attr "type" "bfx")]
4744 )
4745
4746 ;; When the bit position and width add up to 32 we can use a W-reg LSR
4747 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
4748 (define_split
4749   [(set (match_operand:DI 0 "register_operand")
4750         (zero_extract:DI (match_operand:DI 1 "register_operand")
4751                          (match_operand 2
4752                            "aarch64_simd_shift_imm_offset_di")
4753                          (match_operand 3
4754                            "aarch64_simd_shift_imm_di")))]
4755   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
4756              GET_MODE_BITSIZE (DImode) - 1)
4757    && (INTVAL (operands[2]) + INTVAL (operands[3]))
4758        == GET_MODE_BITSIZE (SImode)"
4759   [(set (match_dup 0)
4760         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
4761   {
4762     operands[4] = gen_lowpart (SImode, operands[1]);
4763   }
4764 )
4765
4766 ;; Bitfield Insert (insv)
4767 (define_expand "insv<mode>"
4768   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
4769                           (match_operand 1 "const_int_operand")
4770                           (match_operand 2 "const_int_operand"))
4771         (match_operand:GPI 3 "general_operand"))]
4772   ""
4773 {
4774   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
4775   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
4776   rtx value = operands[3];
4777
4778   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
4779     FAIL;
4780
4781   if (CONST_INT_P (value))
4782     {
4783       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
4784
4785       /* Prefer AND/OR for inserting all zeros or all ones.  */
4786       if ((UINTVAL (value) & mask) == 0
4787            || (UINTVAL (value) & mask) == mask)
4788         FAIL;
4789
4790       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
4791       if (width == 16 && (pos % 16) == 0)
4792         DONE;
4793     }
4794   operands[3] = force_reg (<MODE>mode, value);
4795 })
4796
4797 (define_insn "*insv_reg<mode>"
4798   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4799                           (match_operand 1 "const_int_operand" "n")
4800                           (match_operand 2 "const_int_operand" "n"))
4801         (match_operand:GPI 3 "register_operand" "r"))]
4802   "!(UINTVAL (operands[1]) == 0
4803      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
4804          > GET_MODE_BITSIZE (<MODE>mode)))"
4805   "bfi\\t%<w>0, %<w>3, %2, %1"
4806   [(set_attr "type" "bfm")]
4807 )
4808
4809 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
4810   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4811                           (match_operand 1 "const_int_operand" "n")
4812                           (match_operand 2 "const_int_operand" "n"))
4813         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
4814   "UINTVAL (operands[1]) <= <ALLX:sizen>"
4815   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
4816   [(set_attr "type" "bfm")]
4817 )
4818
4819 (define_insn "*extr_insv_lower_reg<mode>"
4820   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4821                           (match_operand 1 "const_int_operand" "n")
4822                           (const_int 0))
4823         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
4824                           (match_dup 1)
4825                           (match_operand 3 "const_int_operand" "n")))]
4826   "!(UINTVAL (operands[1]) == 0
4827      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
4828          > GET_MODE_BITSIZE (<MODE>mode)))"
4829   "bfxil\\t%<w>0, %<w>2, %3, %1"
4830   [(set_attr "type" "bfm")]
4831 )
4832
4833 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
4834   [(set (match_operand:GPI 0 "register_operand" "=r")
4835         (ashift:GPI (ANY_EXTEND:GPI
4836                      (match_operand:ALLX 1 "register_operand" "r"))
4837                     (match_operand 2 "const_int_operand" "n")))]
4838   "UINTVAL (operands[2]) < <GPI:sizen>"
4839 {
4840   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
4841               ? GEN_INT (<ALLX:sizen>)
4842               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
4843   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4844 }
4845   [(set_attr "type" "bfx")]
4846 )
4847
4848 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
4849
4850 (define_insn "*andim_ashift<mode>_bfiz"
4851   [(set (match_operand:GPI 0 "register_operand" "=r")
4852         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4853                              (match_operand 2 "const_int_operand" "n"))
4854                  (match_operand 3 "const_int_operand" "n")))]
4855   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
4856   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
4857   [(set_attr "type" "bfx")]
4858 )
4859
4860 ;; When the bit position and width of the equivalent extraction add up to 32
4861 ;; we can use a W-reg LSL instruction taking advantage of the implicit
4862 ;; zero-extension of the X-reg.
4863 (define_split
4864   [(set (match_operand:DI 0 "register_operand")
4865         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
4866                              (match_operand 2 "const_int_operand"))
4867                  (match_operand 3 "const_int_operand")))]
4868  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
4869   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
4870       == GET_MODE_BITSIZE (SImode)"
4871   [(set (match_dup 0)
4872         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
4873   {
4874     operands[4] = gen_lowpart (SImode, operands[1]);
4875   }
4876 )
4877
4878 (define_insn "bswap<mode>2"
4879   [(set (match_operand:GPI 0 "register_operand" "=r")
4880         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
4881   ""
4882   "rev\\t%<w>0, %<w>1"
4883   [(set_attr "type" "rev")]
4884 )
4885
4886 (define_insn "bswaphi2"
4887   [(set (match_operand:HI 0 "register_operand" "=r")
4888         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
4889   ""
4890   "rev16\\t%w0, %w1"
4891   [(set_attr "type" "rev")]
4892 )
4893
4894 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
4895 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
4896 ;; each valid permutation.
4897
4898 (define_insn "rev16<mode>2"
4899   [(set (match_operand:GPI 0 "register_operand" "=r")
4900         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4901                                       (const_int 8))
4902                           (match_operand:GPI 3 "const_int_operand" "n"))
4903                  (and:GPI (lshiftrt:GPI (match_dup 1)
4904                                         (const_int 8))
4905                           (match_operand:GPI 2 "const_int_operand" "n"))))]
4906   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4907    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4908   "rev16\\t%<w>0, %<w>1"
4909   [(set_attr "type" "rev")]
4910 )
4911
4912 (define_insn "rev16<mode>2_alt"
4913   [(set (match_operand:GPI 0 "register_operand" "=r")
4914         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
4915                                         (const_int 8))
4916                           (match_operand:GPI 2 "const_int_operand" "n"))
4917                  (and:GPI (ashift:GPI (match_dup 1)
4918                                       (const_int 8))
4919                           (match_operand:GPI 3 "const_int_operand" "n"))))]
4920   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4921    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4922   "rev16\\t%<w>0, %<w>1"
4923   [(set_attr "type" "rev")]
4924 )
4925
4926 ;; zero_extend version of above
4927 (define_insn "*bswapsi2_uxtw"
4928   [(set (match_operand:DI 0 "register_operand" "=r")
4929         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
4930   ""
4931   "rev\\t%w0, %w1"
4932   [(set_attr "type" "rev")]
4933 )
4934
4935 ;; -------------------------------------------------------------------
4936 ;; Floating-point intrinsics
4937 ;; -------------------------------------------------------------------
4938
4939 ;; frint floating-point round to integral standard patterns.
4940 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4941
4942 (define_insn "<frint_pattern><mode>2"
4943   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4944         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4945          FRINT))]
4946   "TARGET_FLOAT"
4947   "frint<frint_suffix>\\t%<s>0, %<s>1"
4948   [(set_attr "type" "f_rint<stype>")]
4949 )
4950
4951 ;; frcvt floating-point round to integer and convert standard patterns.
4952 ;; Expands to lbtrunc, lceil, lfloor, lround.
4953 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
4954   [(set (match_operand:GPI 0 "register_operand" "=r")
4955         (FIXUORS:GPI
4956           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4957            FCVT)))]
4958   "TARGET_FLOAT"
4959   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
4960   [(set_attr "type" "f_cvtf2i")]
4961 )
4962
4963 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
4964   [(set (match_operand:GPI 0 "register_operand" "=r")
4965         (FIXUORS:GPI
4966           (mult:GPF
4967             (match_operand:GPF 1 "register_operand" "w")
4968             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
4969   "TARGET_FLOAT
4970    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
4971                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
4972   {
4973     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
4974     char buf[64];
4975     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
4976     output_asm_insn (buf, operands);
4977     return "";
4978   }
4979   [(set_attr "type" "f_cvtf2i")]
4980 )
4981
4982 ;; fma - expand fma into patterns with the accumulator operand first since
4983 ;; reusing the accumulator results in better register allocation.
4984 ;; The register allocator considers copy preferences in operand order,
4985 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
4986
4987 (define_expand "fma<mode>4"
4988   [(set (match_operand:GPF_F16 0 "register_operand")
4989         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
4990                      (match_operand:GPF_F16 2 "register_operand")
4991                      (match_operand:GPF_F16 3 "register_operand")))]
4992   "TARGET_FLOAT"
4993 )
4994
4995 (define_insn "*aarch64_fma<mode>4"
4996   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4997         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
4998                      (match_operand:GPF_F16 3 "register_operand" "w")
4999                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5000   "TARGET_FLOAT"
5001   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5002   [(set_attr "type" "fmac<stype>")]
5003 )
5004
5005 (define_expand "fnma<mode>4"
5006   [(set (match_operand:GPF_F16 0 "register_operand")
5007         (fma:GPF_F16
5008           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5009           (match_operand:GPF_F16 2 "register_operand")
5010           (match_operand:GPF_F16 3 "register_operand")))]
5011   "TARGET_FLOAT"
5012 )
5013
5014 (define_insn "*aarch64_fnma<mode>4"
5015   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5016         (fma:GPF_F16
5017           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5018           (match_operand:GPF_F16 3 "register_operand" "w")
5019           (match_operand:GPF_F16 1 "register_operand" "w")))]
5020   "TARGET_FLOAT"
5021   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5022   [(set_attr "type" "fmac<stype>")]
5023 )
5024
5025
5026 (define_expand "fms<mode>4"
5027   [(set (match_operand:GPF 0 "register_operand")
5028         (fma:GPF (match_operand:GPF 1 "register_operand")
5029                  (match_operand:GPF 2 "register_operand")
5030                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5031   "TARGET_FLOAT"
5032 )
5033
5034 (define_insn "*aarch64_fms<mode>4"
5035   [(set (match_operand:GPF 0 "register_operand" "=w")
5036         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5037                  (match_operand:GPF 3 "register_operand" "w")
5038                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5039   "TARGET_FLOAT"
5040   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5041   [(set_attr "type" "fmac<s>")]
5042 )
5043
5044 (define_expand "fnms<mode>4"
5045   [(set (match_operand:GPF 0 "register_operand")
5046         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5047                  (match_operand:GPF 2 "register_operand")
5048                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5049   "TARGET_FLOAT"
5050 )
5051
5052 (define_insn "*aarch64_fnms<mode>4"
5053   [(set (match_operand:GPF 0 "register_operand" "=w")
5054         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5055                  (match_operand:GPF 3 "register_operand" "w")
5056                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5057   "TARGET_FLOAT"
5058   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5059   [(set_attr "type" "fmac<s>")]
5060 )
5061
5062 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5063 (define_insn "*aarch64_fnmadd<mode>4"
5064   [(set (match_operand:GPF 0 "register_operand" "=w")
5065         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5066                           (match_operand:GPF 3 "register_operand" "w")
5067                           (match_operand:GPF 1 "register_operand" "w"))))]
5068   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5069   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5070   [(set_attr "type" "fmac<s>")]
5071 )
5072
5073 ;; -------------------------------------------------------------------
5074 ;; Floating-point conversions
5075 ;; -------------------------------------------------------------------
5076
5077 (define_insn "extendsfdf2"
5078   [(set (match_operand:DF 0 "register_operand" "=w")
5079         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5080   "TARGET_FLOAT"
5081   "fcvt\\t%d0, %s1"
5082   [(set_attr "type" "f_cvt")]
5083 )
5084
5085 (define_insn "extendhfsf2"
5086   [(set (match_operand:SF 0 "register_operand" "=w")
5087         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5088   "TARGET_FLOAT"
5089   "fcvt\\t%s0, %h1"
5090   [(set_attr "type" "f_cvt")]
5091 )
5092
5093 (define_insn "extendhfdf2"
5094   [(set (match_operand:DF 0 "register_operand" "=w")
5095         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5096   "TARGET_FLOAT"
5097   "fcvt\\t%d0, %h1"
5098   [(set_attr "type" "f_cvt")]
5099 )
5100
5101 (define_insn "truncdfsf2"
5102   [(set (match_operand:SF 0 "register_operand" "=w")
5103         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5104   "TARGET_FLOAT"
5105   "fcvt\\t%s0, %d1"
5106   [(set_attr "type" "f_cvt")]
5107 )
5108
5109 (define_insn "truncsfhf2"
5110   [(set (match_operand:HF 0 "register_operand" "=w")
5111         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5112   "TARGET_FLOAT"
5113   "fcvt\\t%h0, %s1"
5114   [(set_attr "type" "f_cvt")]
5115 )
5116
5117 (define_insn "truncdfhf2"
5118   [(set (match_operand:HF 0 "register_operand" "=w")
5119         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5120   "TARGET_FLOAT"
5121   "fcvt\\t%h0, %d1"
5122   [(set_attr "type" "f_cvt")]
5123 )
5124
5125 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5126 ;; and making r = w more expensive
5127
5128 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5129   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5130         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5131   "TARGET_FLOAT"
5132   "@
5133    fcvtz<su>\t%<s>0, %<s>1
5134    fcvtz<su>\t%<w>0, %<s>1"
5135   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5136 )
5137
5138 ;; Convert HF -> SI or DI
5139
5140 (define_insn "<optab>_trunchf<GPI:mode>2"
5141   [(set (match_operand:GPI 0 "register_operand" "=r")
5142         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5143   "TARGET_FP_F16INST"
5144   "fcvtz<su>\t%<w>0, %h1"
5145   [(set_attr "type" "f_cvtf2i")]
5146 )
5147
5148 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5149 ;; input in a fp register and output in a integer register
5150
5151 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5152   [(set (match_operand:GPI 0 "register_operand" "=r")
5153         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5154   "TARGET_FLOAT"
5155   "fcvtz<su>\t%<w>0, %<fpw>1"
5156   [(set_attr "type" "f_cvtf2i")]
5157 )
5158
5159 (define_insn "<optab><fcvt_target><GPF:mode>2"
5160   [(set (match_operand:GPF 0 "register_operand" "=w,w")
5161         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
5162   "TARGET_FLOAT"
5163   "@
5164    <su_optab>cvtf\t%<GPF:s>0, %<s>1
5165    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5166   [(set_attr "simd" "yes,no")
5167    (set_attr "fp" "no,yes")
5168    (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
5169 )
5170
5171 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5172   [(set (match_operand:GPF 0 "register_operand" "=w")
5173         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5174   "TARGET_FLOAT"
5175   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5176   [(set_attr "type" "f_cvti2f")]
5177 )
5178
5179 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5180 ;; midend will arrange for an SImode conversion to HFmode to first go
5181 ;; through DFmode, then to HFmode.  But first it will try converting
5182 ;; to DImode then down, which would match our DImode pattern below and
5183 ;; give very poor code-generation.  So, we must provide our own emulation
5184 ;; of the mid-end logic.
5185
5186 (define_insn "aarch64_fp16_<optab><mode>hf2"
5187   [(set (match_operand:HF 0 "register_operand" "=w")
5188         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5189   "TARGET_FP_F16INST"
5190   "<su_optab>cvtf\t%h0, %<w>1"
5191   [(set_attr "type" "f_cvti2f")]
5192 )
5193
5194 (define_expand "<optab>sihf2"
5195   [(set (match_operand:HF 0 "register_operand")
5196         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5197   "TARGET_FLOAT"
5198 {
5199   if (TARGET_FP_F16INST)
5200     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5201   else
5202     {
5203       rtx convert_target = gen_reg_rtx (DFmode);
5204       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5205       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5206     }
5207   DONE;
5208 }
5209 )
5210
5211 ;; For DImode there is no wide enough floating-point mode that we
5212 ;; can convert through natively (TFmode would work, but requires a library
5213 ;; call).  However, we know that any value >= 65504 will be rounded
5214 ;; to infinity on conversion.  This is well within the range of SImode, so
5215 ;; we can:
5216 ;;   Saturate to SImode.
5217 ;;   Convert from that to DFmode
5218 ;;   Convert from that to HFmode (phew!).
5219 ;; Note that the saturation to SImode requires the SIMD extensions.  If
5220 ;; we ever need to provide this pattern where the SIMD extensions are not
5221 ;; available, we would need a different approach.
5222
5223 (define_expand "<optab>dihf2"
5224   [(set (match_operand:HF 0 "register_operand")
5225         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5226   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5227 {
5228   if (TARGET_FP_F16INST)
5229     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5230   else
5231     {
5232       rtx sat_target = gen_reg_rtx (SImode);
5233       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5234       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5235     }
5236
5237   DONE;
5238 }
5239 )
5240
5241 ;; Convert between fixed-point and floating-point (scalar modes)
5242
5243 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
5244   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
5245         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
5246                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5247          FCVT_F2FIXED))]
5248   ""
5249   "@
5250    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
5251    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
5252   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
5253    (set_attr "fp" "yes, *")
5254    (set_attr "simd" "*, yes")]
5255 )
5256
5257 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
5258   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
5259         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
5260                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5261          FCVT_FIXED2F))]
5262   ""
5263   "@
5264    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
5265    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
5266   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
5267    (set_attr "fp" "yes, *")
5268    (set_attr "simd" "*, yes")]
5269 )
5270
5271 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
5272   [(set (match_operand:GPI 0 "register_operand" "=r")
5273         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
5274                      (match_operand:SI 2 "immediate_operand" "i")]
5275          FCVT_F2FIXED))]
5276   "TARGET_FP_F16INST"
5277    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
5278   [(set_attr "type" "f_cvtf2i")]
5279 )
5280
5281 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
5282   [(set (match_operand:HF 0 "register_operand" "=w")
5283         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
5284                     (match_operand:SI 2 "immediate_operand" "i")]
5285          FCVT_FIXED2F))]
5286   "TARGET_FP_F16INST"
5287   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
5288   [(set_attr "type" "f_cvti2f")]
5289 )
5290
5291 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
5292   [(set (match_operand:HI 0 "register_operand" "=w")
5293         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
5294                     (match_operand:SI 2 "immediate_operand" "i")]
5295          FCVT_F2FIXED))]
5296   "TARGET_SIMD"
5297   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
5298   [(set_attr "type" "neon_fp_to_int_s")]
5299 )
5300
5301 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
5302   [(set (match_operand:HF 0 "register_operand" "=w")
5303         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
5304                     (match_operand:SI 2 "immediate_operand" "i")]
5305          FCVT_FIXED2F))]
5306   "TARGET_SIMD"
5307   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
5308   [(set_attr "type" "neon_int_to_fp_s")]
5309 )
5310
5311 ;; -------------------------------------------------------------------
5312 ;; Floating-point arithmetic
5313 ;; -------------------------------------------------------------------
5314
5315 (define_insn "add<mode>3"
5316   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5317         (plus:GPF_F16
5318          (match_operand:GPF_F16 1 "register_operand" "w")
5319          (match_operand:GPF_F16 2 "register_operand" "w")))]
5320   "TARGET_FLOAT"
5321   "fadd\\t%<s>0, %<s>1, %<s>2"
5322   [(set_attr "type" "fadd<stype>")]
5323 )
5324
5325 (define_insn "sub<mode>3"
5326   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5327         (minus:GPF_F16
5328          (match_operand:GPF_F16 1 "register_operand" "w")
5329          (match_operand:GPF_F16 2 "register_operand" "w")))]
5330   "TARGET_FLOAT"
5331   "fsub\\t%<s>0, %<s>1, %<s>2"
5332   [(set_attr "type" "fadd<stype>")]
5333 )
5334
5335 (define_insn "mul<mode>3"
5336   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5337         (mult:GPF_F16
5338          (match_operand:GPF_F16 1 "register_operand" "w")
5339          (match_operand:GPF_F16 2 "register_operand" "w")))]
5340   "TARGET_FLOAT"
5341   "fmul\\t%<s>0, %<s>1, %<s>2"
5342   [(set_attr "type" "fmul<stype>")]
5343 )
5344
5345 (define_insn "*fnmul<mode>3"
5346   [(set (match_operand:GPF 0 "register_operand" "=w")
5347         (mult:GPF
5348                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
5349                  (match_operand:GPF 2 "register_operand" "w")))]
5350   "TARGET_FLOAT && !flag_rounding_math"
5351   "fnmul\\t%<s>0, %<s>1, %<s>2"
5352   [(set_attr "type" "fmul<s>")]
5353 )
5354
5355 (define_insn "*fnmul<mode>3"
5356   [(set (match_operand:GPF 0 "register_operand" "=w")
5357         (neg:GPF (mult:GPF
5358                  (match_operand:GPF 1 "register_operand" "w")
5359                  (match_operand:GPF 2 "register_operand" "w"))))]
5360   "TARGET_FLOAT"
5361   "fnmul\\t%<s>0, %<s>1, %<s>2"
5362   [(set_attr "type" "fmul<s>")]
5363 )
5364
5365 (define_expand "div<mode>3"
5366  [(set (match_operand:GPF_F16 0 "register_operand")
5367        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
5368                     (match_operand:GPF_F16 2 "register_operand")))]
5369  "TARGET_FLOAT"
5370 {
5371   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
5372     DONE;
5373
5374   operands[1] = force_reg (<MODE>mode, operands[1]);
5375 })
5376
5377 (define_insn "*div<mode>3"
5378   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5379         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
5380                      (match_operand:GPF_F16 2 "register_operand" "w")))]
5381   "TARGET_FLOAT"
5382   "fdiv\\t%<s>0, %<s>1, %<s>2"
5383   [(set_attr "type" "fdiv<stype>")]
5384 )
5385
5386 (define_insn "neg<mode>2"
5387   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5388         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5389   "TARGET_FLOAT"
5390   "fneg\\t%<s>0, %<s>1"
5391   [(set_attr "type" "ffarith<stype>")]
5392 )
5393
5394 (define_expand "sqrt<mode>2"
5395   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5396         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5397   "TARGET_FLOAT"
5398 {
5399   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
5400     DONE;
5401 })
5402
5403 (define_insn "*sqrt<mode>2"
5404   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5405         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5406   "TARGET_FLOAT"
5407   "fsqrt\\t%<s>0, %<s>1"
5408   [(set_attr "type" "fsqrt<stype>")]
5409 )
5410
5411 (define_insn "abs<mode>2"
5412   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5413         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5414   "TARGET_FLOAT"
5415   "fabs\\t%<s>0, %<s>1"
5416   [(set_attr "type" "ffarith<stype>")]
5417 )
5418
5419 ;; Given that smax/smin do not specify the result when either input is NaN,
5420 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
5421 ;; for smin.
5422
5423 (define_insn "smax<mode>3"
5424   [(set (match_operand:GPF 0 "register_operand" "=w")
5425         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
5426                   (match_operand:GPF 2 "register_operand" "w")))]
5427   "TARGET_FLOAT"
5428   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
5429   [(set_attr "type" "f_minmax<s>")]
5430 )
5431
5432 (define_insn "smin<mode>3"
5433   [(set (match_operand:GPF 0 "register_operand" "=w")
5434         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
5435                   (match_operand:GPF 2 "register_operand" "w")))]
5436   "TARGET_FLOAT"
5437   "fminnm\\t%<s>0, %<s>1, %<s>2"
5438   [(set_attr "type" "f_minmax<s>")]
5439 )
5440
5441 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
5442 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
5443 ;; which implement the IEEE fmax ()/fmin () functions.
5444 (define_insn "<maxmin_uns><mode>3"
5445   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5446         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
5447                      (match_operand:GPF_F16 2 "register_operand" "w")]
5448                      FMAXMIN_UNS))]
5449   "TARGET_FLOAT"
5450   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
5451   [(set_attr "type" "f_minmax<stype>")]
5452 )
5453
5454 (define_expand "lrint<GPF:mode><GPI:mode>2"
5455   [(match_operand:GPI 0 "register_operand")
5456    (match_operand:GPF 1 "register_operand")]
5457   "TARGET_FLOAT
5458    && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
5459    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
5460 {
5461   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
5462   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
5463   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
5464   DONE;
5465 }
5466 )
5467
5468 ;; For copysign (x, y), we want to generate:
5469 ;;
5470 ;;   LDR d2, #(1 << 63)
5471 ;;   BSL v2.8b, [y], [x]
5472 ;;
5473 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
5474 ;; aarch64_simd_bsldf will select the best suited of these instructions
5475 ;; to generate based on register allocation, and knows how to partially
5476 ;; constant fold based on the values of X and Y, so expand through that.
5477
5478 (define_expand "copysigndf3"
5479   [(match_operand:DF 0 "register_operand")
5480    (match_operand:DF 1 "register_operand")
5481    (match_operand:DF 2 "register_operand")]
5482   "TARGET_FLOAT && TARGET_SIMD"
5483 {
5484   rtx mask = gen_reg_rtx (DImode);
5485   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
5486   emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
5487                                      operands[2], operands[1]));
5488   DONE;
5489 }
5490 )
5491
5492 ;; As above, but we must first get to a 64-bit value if we wish to use
5493 ;; aarch64_simd_bslv2sf.
5494
5495 (define_expand "copysignsf3"
5496   [(match_operand:SF 0 "register_operand")
5497    (match_operand:SF 1 "register_operand")
5498    (match_operand:SF 2 "register_operand")]
5499   "TARGET_FLOAT && TARGET_SIMD"
5500 {
5501   rtx v_bitmask = gen_reg_rtx (V2SImode);
5502
5503   /* Juggle modes to get us in to a vector mode for BSL.  */
5504   rtx op1 = lowpart_subreg (DImode, operands[1], SFmode);
5505   rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
5506   rtx tmp = gen_reg_rtx (V2SFmode);
5507   emit_move_insn (v_bitmask,
5508                   aarch64_simd_gen_const_vector_dup (V2SImode,
5509                                                      HOST_WIDE_INT_M1U << 31));
5510   emit_insn (gen_aarch64_simd_bslv2sf (tmp, v_bitmask, op2, op1));
5511   emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
5512   DONE;
5513 }
5514 )
5515
5516 ;; For xorsign (x, y), we want to generate:
5517 ;;
5518 ;; LDR   d2, #1<<63
5519 ;; AND   v3.8B, v1.8B, v2.8B
5520 ;; EOR   v0.8B, v0.8B, v3.8B
5521 ;;
5522
5523 (define_expand "xorsign<mode>3"
5524   [(match_operand:GPF 0 "register_operand")
5525    (match_operand:GPF 1 "register_operand")
5526    (match_operand:GPF 2 "register_operand")]
5527   "TARGET_FLOAT && TARGET_SIMD"
5528 {
5529
5530   machine_mode imode = <V_INT_EQUIV>mode;
5531   rtx mask = gen_reg_rtx (imode);
5532   rtx op1x = gen_reg_rtx (imode);
5533   rtx op2x = gen_reg_rtx (imode);
5534
5535   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
5536   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
5537                                                      imode)));
5538
5539   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
5540                                     lowpart_subreg (imode, operands[2],
5541                                                     <MODE>mode)));
5542   emit_insn (gen_xor<v_int_equiv>3 (op1x,
5543                                     lowpart_subreg (imode, operands[1],
5544                                                     <MODE>mode),
5545                                     op2x));
5546   emit_move_insn (operands[0],
5547                   lowpart_subreg (<MODE>mode, op1x, imode));
5548   DONE;
5549 }
5550 )
5551
5552 ;; -------------------------------------------------------------------
5553 ;; Reload support
5554 ;; -------------------------------------------------------------------
5555 ;; Reload Scalar Floating point modes from constant pool.
5556 ;; The AArch64 port doesn't have __int128 constant move support.
5557 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
5558  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
5559        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
5560   (clobber (match_operand:P 2 "register_operand" "=&r"))]
5561  "TARGET_FLOAT"
5562  {
5563    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5564    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
5565    DONE;
5566  }
5567 )
5568
5569 ;; Reload Vector modes from constant pool.
5570 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
5571  [(set (match_operand:VALL 0 "register_operand" "=w")
5572        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
5573   (clobber (match_operand:P 2 "register_operand" "=&r"))]
5574  "TARGET_FLOAT"
5575  {
5576    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5577    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
5578    DONE;
5579  }
5580 )
5581
5582 (define_expand "aarch64_reload_mov<mode>"
5583   [(set (match_operand:TX 0 "register_operand" "=w")
5584         (match_operand:TX 1 "register_operand" "w"))
5585    (clobber (match_operand:DI 2 "register_operand" "=&r"))
5586   ]
5587   "TARGET_FLOAT"
5588   {
5589     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
5590     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
5591     gen_aarch64_movtilow_tilow (op0, op1);
5592     gen_aarch64_movdi_tihigh (operands[2], op1);
5593     gen_aarch64_movtihigh_di (op0, operands[2]);
5594     DONE;
5595   }
5596 )
5597
5598 ;; The following secondary reload helpers patterns are invoked
5599 ;; after or during reload as we don't want these patterns to start
5600 ;; kicking in during the combiner.
5601
5602 (define_insn "aarch64_movdi_<mode>low"
5603   [(set (match_operand:DI 0 "register_operand" "=r")
5604         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5605                          (const_int 64) (const_int 0)))]
5606   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5607   "fmov\\t%x0, %d1"
5608   [(set_attr "type" "f_mrc")
5609    (set_attr "length" "4")
5610   ])
5611
5612 (define_insn "aarch64_movdi_<mode>high"
5613   [(set (match_operand:DI 0 "register_operand" "=r")
5614         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5615                          (const_int 64) (const_int 64)))]
5616   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5617   "fmov\\t%x0, %1.d[1]"
5618   [(set_attr "type" "f_mrc")
5619    (set_attr "length" "4")
5620   ])
5621
5622 (define_insn "aarch64_mov<mode>high_di"
5623   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
5624                          (const_int 64) (const_int 64))
5625         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5626   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5627   "fmov\\t%0.d[1], %x1"
5628   [(set_attr "type" "f_mcr")
5629    (set_attr "length" "4")
5630   ])
5631
5632 (define_insn "aarch64_mov<mode>low_di"
5633   [(set (match_operand:TX 0 "register_operand" "=w")
5634         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5635   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5636   "fmov\\t%d0, %x1"
5637   [(set_attr "type" "f_mcr")
5638    (set_attr "length" "4")
5639   ])
5640
5641 (define_insn "aarch64_movtilow_tilow"
5642   [(set (match_operand:TI 0 "register_operand" "=w")
5643         (zero_extend:TI
5644           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
5645   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5646   "fmov\\t%d0, %d1"
5647   [(set_attr "type" "fmov")
5648    (set_attr "length" "4")
5649   ])
5650
5651 ;; There is a deliberate reason why the parameters of high and lo_sum's
5652 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
5653 ;; and lo_sum's to be used with the labels defining the jump tables in
5654 ;; rodata section.
5655
5656 (define_expand "add_losym"
5657   [(set (match_operand 0 "register_operand" "=r")
5658         (lo_sum (match_operand 1 "register_operand" "r")
5659                 (match_operand 2 "aarch64_valid_symref" "S")))]
5660   ""
5661 {
5662   machine_mode mode = GET_MODE (operands[0]);
5663
5664   emit_insn ((mode == DImode
5665               ? gen_add_losym_di
5666               : gen_add_losym_si) (operands[0],
5667                                    operands[1],
5668                                    operands[2]));
5669   DONE;
5670 })
5671
5672 (define_insn "add_losym_<mode>"
5673   [(set (match_operand:P 0 "register_operand" "=r")
5674         (lo_sum:P (match_operand:P 1 "register_operand" "r")
5675                   (match_operand 2 "aarch64_valid_symref" "S")))]
5676   ""
5677   "add\\t%<w>0, %<w>1, :lo12:%c2"
5678   [(set_attr "type" "alu_imm")]
5679 )
5680
5681 (define_insn "ldr_got_small_<mode>"
5682   [(set (match_operand:PTR 0 "register_operand" "=r")
5683         (unspec:PTR [(mem:PTR (lo_sum:PTR
5684                               (match_operand:PTR 1 "register_operand" "r")
5685                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5686                     UNSPEC_GOTSMALLPIC))]
5687   ""
5688   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
5689   [(set_attr "type" "load_<ldst_sz>")]
5690 )
5691
5692 (define_insn "ldr_got_small_sidi"
5693   [(set (match_operand:DI 0 "register_operand" "=r")
5694         (zero_extend:DI
5695          (unspec:SI [(mem:SI (lo_sum:DI
5696                              (match_operand:DI 1 "register_operand" "r")
5697                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5698                     UNSPEC_GOTSMALLPIC)))]
5699   "TARGET_ILP32"
5700   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
5701   [(set_attr "type" "load_4")]
5702 )
5703
5704 (define_insn "ldr_got_small_28k_<mode>"
5705   [(set (match_operand:PTR 0 "register_operand" "=r")
5706         (unspec:PTR [(mem:PTR (lo_sum:PTR
5707                               (match_operand:PTR 1 "register_operand" "r")
5708                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5709                     UNSPEC_GOTSMALLPIC28K))]
5710   ""
5711   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
5712   [(set_attr "type" "load_<ldst_sz>")]
5713 )
5714
5715 (define_insn "ldr_got_small_28k_sidi"
5716   [(set (match_operand:DI 0 "register_operand" "=r")
5717         (zero_extend:DI
5718          (unspec:SI [(mem:SI (lo_sum:DI
5719                              (match_operand:DI 1 "register_operand" "r")
5720                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5721                     UNSPEC_GOTSMALLPIC28K)))]
5722   "TARGET_ILP32"
5723   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
5724   [(set_attr "type" "load_4")]
5725 )
5726
5727 (define_insn "ldr_got_tiny"
5728   [(set (match_operand:DI 0 "register_operand" "=r")
5729         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
5730                    UNSPEC_GOTTINYPIC))]
5731   ""
5732   "ldr\\t%0, %L1"
5733   [(set_attr "type" "load_8")]
5734 )
5735
5736 (define_insn "aarch64_load_tp_hard"
5737   [(set (match_operand:DI 0 "register_operand" "=r")
5738         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
5739   ""
5740   "mrs\\t%0, tpidr_el0"
5741   [(set_attr "type" "mrs")]
5742 )
5743
5744 ;; The TLS ABI specifically requires that the compiler does not schedule
5745 ;; instructions in the TLS stubs, in order to enable linker relaxation.
5746 ;; Therefore we treat the stubs as an atomic sequence.
5747 (define_expand "tlsgd_small_<mode>"
5748  [(parallel [(set (match_operand 0 "register_operand" "")
5749                   (call (mem:DI (match_dup 2)) (const_int 1)))
5750              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
5751              (clobber (reg:DI LR_REGNUM))])]
5752  ""
5753 {
5754   operands[2] = aarch64_tls_get_addr ();
5755 })
5756
5757 (define_insn "*tlsgd_small_<mode>"
5758   [(set (match_operand 0 "register_operand" "")
5759         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
5760    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
5761    (clobber (reg:DI LR_REGNUM))
5762   ]
5763   ""
5764   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
5765   [(set_attr "type" "call")
5766    (set_attr "length" "16")])
5767
5768 (define_insn "tlsie_small_<mode>"
5769   [(set (match_operand:PTR 0 "register_operand" "=r")
5770         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5771                    UNSPEC_GOTSMALLTLS))]
5772   ""
5773   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
5774   [(set_attr "type" "load_4")
5775    (set_attr "length" "8")]
5776 )
5777
5778 (define_insn "tlsie_small_sidi"
5779   [(set (match_operand:DI 0 "register_operand" "=r")
5780         (zero_extend:DI
5781           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5782                       UNSPEC_GOTSMALLTLS)))]
5783   ""
5784   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
5785   [(set_attr "type" "load_4")
5786    (set_attr "length" "8")]
5787 )
5788
5789 (define_insn "tlsie_tiny_<mode>"
5790   [(set (match_operand:PTR 0 "register_operand" "=&r")
5791         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
5792                      (match_operand:PTR 2 "register_operand" "r")]
5793                    UNSPEC_GOTTINYTLS))]
5794   ""
5795   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
5796   [(set_attr "type" "multiple")
5797    (set_attr "length" "8")]
5798 )
5799
5800 (define_insn "tlsie_tiny_sidi"
5801   [(set (match_operand:DI 0 "register_operand" "=&r")
5802         (zero_extend:DI
5803           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
5804                       (match_operand:DI 2 "register_operand" "r")
5805                       ]
5806                       UNSPEC_GOTTINYTLS)))]
5807   ""
5808   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5809   [(set_attr "type" "multiple")
5810    (set_attr "length" "8")]
5811 )
5812
5813 (define_insn "tlsle12_<mode>"
5814   [(set (match_operand:P 0 "register_operand" "=r")
5815         (unspec:P [(match_operand:P 1 "register_operand" "r")
5816                    (match_operand 2 "aarch64_tls_le_symref" "S")]
5817                    UNSPEC_TLSLE12))]
5818   ""
5819   "add\\t%<w>0, %<w>1, #%L2";
5820   [(set_attr "type" "alu_sreg")
5821    (set_attr "length" "4")]
5822 )
5823
5824 (define_insn "tlsle24_<mode>"
5825   [(set (match_operand:P 0 "register_operand" "=r")
5826         (unspec:P [(match_operand:P 1 "register_operand" "r")
5827                    (match_operand 2 "aarch64_tls_le_symref" "S")]
5828                    UNSPEC_TLSLE24))]
5829   ""
5830   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
5831   [(set_attr "type" "multiple")
5832    (set_attr "length" "8")]
5833 )
5834
5835 (define_insn "tlsle32_<mode>"
5836   [(set (match_operand:P 0 "register_operand" "=r")
5837         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5838                    UNSPEC_TLSLE32))]
5839   ""
5840   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5841   [(set_attr "type" "multiple")
5842    (set_attr "length" "8")]
5843 )
5844
5845 (define_insn "tlsle48_<mode>"
5846   [(set (match_operand:P 0 "register_operand" "=r")
5847         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5848                    UNSPEC_TLSLE48))]
5849   ""
5850   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5851   [(set_attr "type" "multiple")
5852    (set_attr "length" "12")]
5853 )
5854
5855 (define_expand "tlsdesc_small_<mode>"
5856   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
5857   "TARGET_TLS_DESC"
5858   {
5859     if (TARGET_SVE)
5860       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
5861     else
5862       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
5863     DONE;
5864   }
5865 )
5866
5867 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
5868 ;; R0 and LR.
5869 (define_insn "tlsdesc_small_advsimd_<mode>"
5870   [(set (reg:PTR R0_REGNUM)
5871         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
5872                     UNSPEC_TLSDESC))
5873    (clobber (reg:DI LR_REGNUM))
5874    (clobber (reg:CC CC_REGNUM))
5875    (clobber (match_scratch:DI 1 "=r"))]
5876   "TARGET_TLS_DESC && !TARGET_SVE"
5877   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
5878   [(set_attr "type" "call")
5879    (set_attr "length" "16")])
5880
5881 ;; For SVE, model tlsdesc calls as clobbering all vector and predicate
5882 ;; registers, on top of the usual R0 and LR.  In reality the calls
5883 ;; preserve the low 128 bits of the vector registers, but we don't
5884 ;; yet have a way of representing that in the instruction pattern.
5885 (define_insn "tlsdesc_small_sve_<mode>"
5886   [(set (reg:PTR R0_REGNUM)
5887         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
5888                     UNSPEC_TLSDESC))
5889    (clobber (reg:DI LR_REGNUM))
5890    (clobber (reg:CC CC_REGNUM))
5891    (clobber (reg:XI V0_REGNUM))
5892    (clobber (reg:XI V4_REGNUM))
5893    (clobber (reg:XI V8_REGNUM))
5894    (clobber (reg:XI V12_REGNUM))
5895    (clobber (reg:XI V16_REGNUM))
5896    (clobber (reg:XI V20_REGNUM))
5897    (clobber (reg:XI V24_REGNUM))
5898    (clobber (reg:XI V28_REGNUM))
5899    (clobber (reg:VNx2BI P0_REGNUM))
5900    (clobber (reg:VNx2BI P1_REGNUM))
5901    (clobber (reg:VNx2BI P2_REGNUM))
5902    (clobber (reg:VNx2BI P3_REGNUM))
5903    (clobber (reg:VNx2BI P4_REGNUM))
5904    (clobber (reg:VNx2BI P5_REGNUM))
5905    (clobber (reg:VNx2BI P6_REGNUM))
5906    (clobber (reg:VNx2BI P7_REGNUM))
5907    (clobber (reg:VNx2BI P8_REGNUM))
5908    (clobber (reg:VNx2BI P9_REGNUM))
5909    (clobber (reg:VNx2BI P10_REGNUM))
5910    (clobber (reg:VNx2BI P11_REGNUM))
5911    (clobber (reg:VNx2BI P12_REGNUM))
5912    (clobber (reg:VNx2BI P13_REGNUM))
5913    (clobber (reg:VNx2BI P14_REGNUM))
5914    (clobber (reg:VNx2BI P15_REGNUM))
5915    (clobber (match_scratch:DI 1 "=r"))]
5916   "TARGET_TLS_DESC && TARGET_SVE"
5917   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
5918   [(set_attr "type" "call")
5919    (set_attr "length" "16")])
5920
5921 (define_insn "stack_tie"
5922   [(set (mem:BLK (scratch))
5923         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
5924                      (match_operand:DI 1 "register_operand" "rk")]
5925                     UNSPEC_PRLG_STK))]
5926   ""
5927   ""
5928   [(set_attr "length" "0")]
5929 )
5930
5931 ;; Pointer authentication patterns are always provided.  In architecture
5932 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
5933 ;; This lets the user write portable software which authenticates pointers
5934 ;; when run on something which implements ARMv8.3-A, and which runs
5935 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
5936 ;; implemented.
5937
5938 ;; Signing/Authenticating R30 using SP as the salt.
5939
5940 (define_insn "<pauth_mnem_prefix>sp"
5941   [(set (reg:DI R30_REGNUM)
5942         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
5943   ""
5944   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
5945 )
5946
5947 ;; Signing/Authenticating X17 using X16 as the salt.
5948
5949 (define_insn "<pauth_mnem_prefix>1716"
5950   [(set (reg:DI R17_REGNUM)
5951         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
5952   ""
5953   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
5954 )
5955
5956 ;; Stripping the signature in R30.
5957
5958 (define_insn "xpaclri"
5959   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
5960   ""
5961   "hint\t7 // xpaclri"
5962 )
5963
5964 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
5965 ;; all of memory.  This blocks insns from being moved across this point.
5966
5967 (define_insn "blockage"
5968   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
5969   ""
5970   ""
5971   [(set_attr "length" "0")
5972    (set_attr "type" "block")]
5973 )
5974
5975 (define_insn "probe_stack_range"
5976   [(set (match_operand:DI 0 "register_operand" "=r")
5977         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
5978                              (match_operand:DI 2 "register_operand" "r")]
5979                               UNSPECV_PROBE_STACK_RANGE))]
5980   ""
5981 {
5982   return aarch64_output_probe_stack_range (operands[0], operands[2]);
5983 }
5984   [(set_attr "length" "32")]
5985 )
5986
5987 ;; Named pattern for expanding thread pointer reference.
5988 (define_expand "get_thread_pointerdi"
5989   [(match_operand:DI 0 "register_operand" "=r")]
5990   ""
5991 {
5992   rtx tmp = aarch64_load_tp (operands[0]);
5993   if (tmp != operands[0])
5994     emit_move_insn (operands[0], tmp);
5995   DONE;
5996 })
5997
5998 ;; Named patterns for stack smashing protection.
5999 (define_expand "stack_protect_set"
6000   [(match_operand 0 "memory_operand")
6001    (match_operand 1 "memory_operand")]
6002   ""
6003 {
6004   machine_mode mode = GET_MODE (operands[0]);
6005
6006   emit_insn ((mode == DImode
6007               ? gen_stack_protect_set_di
6008               : gen_stack_protect_set_si) (operands[0], operands[1]));
6009   DONE;
6010 })
6011
6012 (define_insn "stack_protect_set_<mode>"
6013   [(set (match_operand:PTR 0 "memory_operand" "=m")
6014         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6015          UNSPEC_SP_SET))
6016    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6017   ""
6018   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
6019   [(set_attr "length" "12")
6020    (set_attr "type" "multiple")])
6021
6022 (define_expand "stack_protect_test"
6023   [(match_operand 0 "memory_operand")
6024    (match_operand 1 "memory_operand")
6025    (match_operand 2)]
6026   ""
6027 {
6028   rtx result;
6029   machine_mode mode = GET_MODE (operands[0]);
6030
6031   result = gen_reg_rtx(mode);
6032
6033   emit_insn ((mode == DImode
6034               ? gen_stack_protect_test_di
6035               : gen_stack_protect_test_si) (result,
6036                                             operands[0],
6037                                             operands[1]));
6038
6039   if (mode == DImode)
6040     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6041                                     result, const0_rtx, operands[2]));
6042   else
6043     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6044                                     result, const0_rtx, operands[2]));
6045   DONE;
6046 })
6047
6048 (define_insn "stack_protect_test_<mode>"
6049   [(set (match_operand:PTR 0 "register_operand" "=r")
6050         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
6051                      (match_operand:PTR 2 "memory_operand" "m")]
6052          UNSPEC_SP_TEST))
6053    (clobber (match_scratch:PTR 3 "=&r"))]
6054   ""
6055   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
6056   [(set_attr "length" "12")
6057    (set_attr "type" "multiple")])
6058
6059 ;; Write Floating-point Control Register.
6060 (define_insn "set_fpcr"
6061   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
6062   ""
6063   "msr\\tfpcr, %0"
6064   [(set_attr "type" "mrs")])
6065
6066 ;; Read Floating-point Control Register.
6067 (define_insn "get_fpcr"
6068   [(set (match_operand:SI 0 "register_operand" "=r")
6069         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
6070   ""
6071   "mrs\\t%0, fpcr"
6072   [(set_attr "type" "mrs")])
6073
6074 ;; Write Floating-point Status Register.
6075 (define_insn "set_fpsr"
6076   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
6077   ""
6078   "msr\\tfpsr, %0"
6079   [(set_attr "type" "mrs")])
6080
6081 ;; Read Floating-point Status Register.
6082 (define_insn "get_fpsr"
6083   [(set (match_operand:SI 0 "register_operand" "=r")
6084         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
6085   ""
6086   "mrs\\t%0, fpsr"
6087   [(set_attr "type" "mrs")])
6088
6089
6090 ;; Define the subtract-one-and-jump insns so loop.c
6091 ;; knows what to generate.
6092 (define_expand "doloop_end"
6093   [(use (match_operand 0 "" ""))      ; loop pseudo
6094    (use (match_operand 1 "" ""))]     ; label
6095   "optimize > 0 && flag_modulo_sched"
6096 {
6097   rtx s0;
6098   rtx bcomp;
6099   rtx loc_ref;
6100   rtx cc_reg;
6101   rtx insn;
6102   rtx cmp;
6103
6104   /* Currently SMS relies on the do-loop pattern to recognize loops
6105      where (1) the control part consists of all insns defining and/or
6106      using a certain 'count' register and (2) the loop count can be
6107      adjusted by modifying this register prior to the loop.
6108      ??? The possible introduction of a new block to initialize the
6109      new IV can potentially affect branch optimizations.  */
6110
6111   if (GET_MODE (operands[0]) != DImode)
6112     FAIL;
6113
6114   s0 = operands [0];
6115   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
6116
6117   cmp = XVECEXP (PATTERN (insn), 0, 0);
6118   cc_reg = SET_DEST (cmp);
6119   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6120   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6121   emit_jump_insn (gen_rtx_SET (pc_rtx,
6122                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6123                                                      loc_ref, pc_rtx)));
6124   DONE;
6125 })
6126
6127 ;; Helper for aarch64.c code.
6128 (define_expand "set_clobber_cc"
6129   [(parallel [(set (match_operand 0)
6130                    (match_operand 1))
6131               (clobber (reg:CC CC_REGNUM))])])
6132
6133 ;; AdvSIMD Stuff
6134 (include "aarch64-simd.md")
6135
6136 ;; Atomic Operations
6137 (include "atomics.md")
6138
6139 ;; ldp/stp peephole patterns
6140 (include "aarch64-ldpstp.md")
6141
6142 ;; SVE.
6143 (include "aarch64-sve.md")