gcc/config/aarch64/aarch64.md

   1 ;; Machine description for AArch64 architecture.
   2 ;; Copyright (C) 2009-2018 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21 ;; Register numbers
  22 (define_constants
  23   [
  24     (R0_REGNUM          0)
  25     (R1_REGNUM          1)
  26     (R2_REGNUM          2)
  27     (R3_REGNUM          3)
  28     (R4_REGNUM          4)
  29     (R5_REGNUM          5)
  30     (R6_REGNUM          6)
  31     (R7_REGNUM          7)
  32     (R8_REGNUM          8)
  33     (R9_REGNUM          9)
  34     (R10_REGNUM         10)
  35     (R11_REGNUM         11)
  36     (R12_REGNUM         12)
  37     (R13_REGNUM         13)
  38     (R14_REGNUM         14)
  39     (R15_REGNUM         15)
  40     (R16_REGNUM         16)
  41     (IP0_REGNUM         16)
  42     (R17_REGNUM         17)
  43     (IP1_REGNUM         17)
  44     (R18_REGNUM         18)
  45     (R19_REGNUM         19)
  46     (R20_REGNUM         20)
  47     (R21_REGNUM         21)
  48     (R22_REGNUM         22)
  49     (R23_REGNUM         23)
  50     (R24_REGNUM         24)
  51     (R25_REGNUM         25)
  52     (R26_REGNUM         26)
  53     (R27_REGNUM         27)
  54     (R28_REGNUM         28)
  55     (R29_REGNUM         29)
  56     (R30_REGNUM         30)
  57     (LR_REGNUM          30)
  58     (SP_REGNUM          31)
  59     (V0_REGNUM          32)
  60     (V1_REGNUM          33)
  61     (V2_REGNUM          34)
  62     (V3_REGNUM          35)
  63     (V4_REGNUM          36)
  64     (V5_REGNUM          37)
  65     (V6_REGNUM          38)
  66     (V7_REGNUM          39)
  67     (V8_REGNUM          40)
  68     (V9_REGNUM          41)
  69     (V10_REGNUM         42)
  70     (V11_REGNUM         43)
  71     (V12_REGNUM         44)
  72     (V13_REGNUM         45)
  73     (V14_REGNUM         46)
  74     (V15_REGNUM         47)
  75     (V16_REGNUM         48)
  76     (V17_REGNUM         49)
  77     (V18_REGNUM         50)
  78     (V19_REGNUM         51)
  79     (V20_REGNUM         52)
  80     (V21_REGNUM         53)
  81     (V22_REGNUM         54)
  82     (V23_REGNUM         55)
  83     (V24_REGNUM         56)
  84     (V25_REGNUM         57)
  85     (V26_REGNUM         58)
  86     (V27_REGNUM         59)
  87     (V28_REGNUM         60)
  88     (V29_REGNUM         61)
  89     (V30_REGNUM         62)
  90     (V31_REGNUM         63)
  91     (LAST_SAVED_REGNUM  63)
  92     (SFP_REGNUM         64)
  93     (AP_REGNUM          65)
  94     (CC_REGNUM          66)
  95     ;; Defined only to make the DWARF description simpler.
  96     (VG_REGNUM          67)
  97     (P0_REGNUM          68)
  98     (P1_REGNUM          69)
  99     (P2_REGNUM          70)
 100     (P3_REGNUM          71)
 101     (P4_REGNUM          72)
 102     (P5_REGNUM          73)
 103     (P6_REGNUM          74)
 104     (P7_REGNUM          75)
 105     (P8_REGNUM          76)
 106     (P9_REGNUM          77)
 107     (P10_REGNUM         78)
 108     (P11_REGNUM         79)
 109     (P12_REGNUM         80)
 110     (P13_REGNUM         81)
 111     (P14_REGNUM         82)
 112     (P15_REGNUM         83)
 113     ;; A couple of call-clobbered registers that we need to reserve when
 114     ;; tracking speculation this is not ABI, so is subject to change.
 115     (SPECULATION_TRACKER_REGNUM 15)
 116     (SPECULATION_SCRATCH_REGNUM 14)
 117   ]
 118 )
 119
 120 (define_c_enum "unspec" [
 121     UNSPEC_AUTI1716
 122     UNSPEC_AUTISP
 123     UNSPEC_CASESI
 124     UNSPEC_CRC32B
 125     UNSPEC_CRC32CB
 126     UNSPEC_CRC32CH
 127     UNSPEC_CRC32CW
 128     UNSPEC_CRC32CX
 129     UNSPEC_CRC32H
 130     UNSPEC_CRC32W
 131     UNSPEC_CRC32X
 132     UNSPEC_FCVTZS
 133     UNSPEC_FCVTZU
 134     UNSPEC_URECPE
 135     UNSPEC_FRECPE
 136     UNSPEC_FRECPS
 137     UNSPEC_FRECPX
 138     UNSPEC_FRINTA
 139     UNSPEC_FRINTI
 140     UNSPEC_FRINTM
 141     UNSPEC_FRINTN
 142     UNSPEC_FRINTP
 143     UNSPEC_FRINTX
 144     UNSPEC_FRINTZ
 145     UNSPEC_GOTSMALLPIC
 146     UNSPEC_GOTSMALLPIC28K
 147     UNSPEC_GOTSMALLTLS
 148     UNSPEC_GOTTINYPIC
 149     UNSPEC_GOTTINYTLS
 150     UNSPEC_LD1
 151     UNSPEC_LD2
 152     UNSPEC_LD2_DREG
 153     UNSPEC_LD2_DUP
 154     UNSPEC_LD3
 155     UNSPEC_LD3_DREG
 156     UNSPEC_LD3_DUP
 157     UNSPEC_LD4
 158     UNSPEC_LD4_DREG
 159     UNSPEC_LD4_DUP
 160     UNSPEC_LD2_LANE
 161     UNSPEC_LD3_LANE
 162     UNSPEC_LD4_LANE
 163     UNSPEC_MB
 164     UNSPEC_NOP
 165     UNSPEC_PACI1716
 166     UNSPEC_PACISP
 167     UNSPEC_PRLG_STK
 168     UNSPEC_REV
 169     UNSPEC_RBIT
 170     UNSPEC_SABAL
 171     UNSPEC_SABDL2
 172     UNSPEC_SADALP
 173     UNSPEC_SCVTF
 174     UNSPEC_SISD_NEG
 175     UNSPEC_SISD_SSHL
 176     UNSPEC_SISD_USHL
 177     UNSPEC_SSHL_2S
 178     UNSPEC_ST1
 179     UNSPEC_ST2
 180     UNSPEC_ST3
 181     UNSPEC_ST4
 182     UNSPEC_ST2_LANE
 183     UNSPEC_ST3_LANE
 184     UNSPEC_ST4_LANE
 185     UNSPEC_TLS
 186     UNSPEC_TLSDESC
 187     UNSPEC_TLSLE12
 188     UNSPEC_TLSLE24
 189     UNSPEC_TLSLE32
 190     UNSPEC_TLSLE48
 191     UNSPEC_UABAL
 192     UNSPEC_UABDL2
 193     UNSPEC_UADALP
 194     UNSPEC_UCVTF
 195     UNSPEC_USHL_2S
 196     UNSPEC_VSTRUCTDUMMY
 197     UNSPEC_SP_SET
 198     UNSPEC_SP_TEST
 199     UNSPEC_RSQRT
 200     UNSPEC_RSQRTE
 201     UNSPEC_RSQRTS
 202     UNSPEC_NZCV
 203     UNSPEC_XPACLRI
 204     UNSPEC_LD1_SVE
 205     UNSPEC_ST1_SVE
 206     UNSPEC_LD1RQ
 207     UNSPEC_LD1_GATHER
 208     UNSPEC_ST1_SCATTER
 209     UNSPEC_MERGE_PTRUE
 210     UNSPEC_PTEST_PTRUE
 211     UNSPEC_UNPACKSHI
 212     UNSPEC_UNPACKUHI
 213     UNSPEC_UNPACKSLO
 214     UNSPEC_UNPACKULO
 215     UNSPEC_PACK
 216     UNSPEC_FLOAT_CONVERT
 217     UNSPEC_WHILE_LO
 218     UNSPEC_LDN
 219     UNSPEC_STN
 220     UNSPEC_INSR
 221     UNSPEC_CLASTB
 222     UNSPEC_FADDA
 223     UNSPEC_REV_SUBREG
 224     UNSPEC_SPECULATION_TRACKER
 225 ])
 226
 227 (define_c_enum "unspecv" [
 228     UNSPECV_EH_RETURN           ; Represent EH_RETURN
 229     UNSPECV_GET_FPCR            ; Represent fetch of FPCR content.
 230     UNSPECV_SET_FPCR            ; Represent assign of FPCR content.
 231     UNSPECV_GET_FPSR            ; Represent fetch of FPSR content.
 232     UNSPECV_SET_FPSR            ; Represent assign of FPSR content.
 233     UNSPECV_BLOCKAGE            ; Represent a blockage
 234     UNSPECV_PROBE_STACK_RANGE   ; Represent stack range probing.
 235     UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
 236   ]
 237 )
 238
 239 ;; If further include files are added the defintion of MD_INCLUDES
 240 ;; must be updated.
 241
 242 (include "constraints.md")
 243 (include "predicates.md")
 244 (include "iterators.md")
 245
 246 ;; -------------------------------------------------------------------
 247 ;; Instruction types and attributes
 248 ;; -------------------------------------------------------------------
 249
 250 ; The "type" attribute is included here from AArch32 backend to be able
 251 ; to share pipeline descriptions.
 252 (include "../arm/types.md")
 253
 254 ;; It is important to set the fp or simd attributes to yes when a pattern
 255 ;; alternative uses the FP or SIMD register files, usually signified by use of
 256 ;; the 'w' constraint.  This will ensure that the alternative will be
 257 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
 258 ;; architecture extensions.  If all the alternatives in a pattern use the
 259 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
 260 ;; or TARGET_SIMD.
 261
 262 ;; Attributes of the architecture required to support the instruction (or
 263 ;; alternative). This attribute is used to compute attribute "enabled", use type
 264 ;; "any" to enable an alternative in all cases.
 265
 266 (define_enum "arches" [ any rcpc8_4 fp simd sve fp16])
 267
 268 (define_enum_attr "arch" "arches" (const_string "any"))
 269
 270 ;; [For compatibility with Arm in pipeline models]
 271 ;; Attribute that specifies whether or not the instruction touches fp
 272 ;; registers.
 273 ;; Note that this attribute is not used anywhere in either the arm or aarch64
 274 ;; backends except in the scheduling description for xgene1.  In that
 275 ;; scheduling description this attribute is used to subclass the load_4 and
 276 ;; load_8 types.
 277 (define_attr "fp" "no,yes"
 278   (if_then_else
 279     (eq_attr "arch" "fp")
 280     (const_string "yes")
 281     (const_string "no")))
 282
 283 (define_attr "arch_enabled" "no,yes"
 284   (if_then_else
 285     (ior
 286         (eq_attr "arch" "any")
 287
 288         (and (eq_attr "arch" "rcpc8_4")
 289              (match_test "AARCH64_ISA_RCPC8_4"))
 290
 291         (and (eq_attr "arch" "fp")
 292              (match_test "TARGET_FLOAT"))
 293
 294         (and (eq_attr "arch" "simd")
 295              (match_test "TARGET_SIMD"))
 296
 297         (and (eq_attr "arch" "fp16")
 298              (match_test "TARGET_FP_F16INST"))
 299
 300         (and (eq_attr "arch" "sve")
 301              (match_test "TARGET_SVE")))
 302     (const_string "yes")
 303     (const_string "no")))
 304
 305 ;; Attribute that controls whether an alternative is enabled or not.
 306 ;; Currently it is only used to disable alternatives which touch fp or simd
 307 ;; registers when -mgeneral-regs-only is specified or to require a special
 308 ;; architecture support.
 309 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
 310
 311 ;; Attribute that specifies whether we are dealing with a branch to a
 312 ;; label that is far away, i.e. further away than the maximum/minimum
 313 ;; representable in a signed 21-bits number.
 314 ;; 0 :=: no
 315 ;; 1 :=: yes
 316 (define_attr "far_branch" "" (const_int 0))
 317
 318 ;; Attribute that specifies whether the alternative uses MOVPRFX.
 319 (define_attr "movprfx" "no,yes" (const_string "no"))
 320
 321 (define_attr "length" ""
 322   (cond [(eq_attr "movprfx" "yes")
 323            (const_int 8)
 324         ] (const_int 4)))
 325
 326 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
 327 ;; no predicated insns.
 328 (define_attr "predicated" "yes,no" (const_string "no"))
 329
 330 ;; Set to true on an insn that requires the speculation tracking state to be
 331 ;; in the tracking register before the insn issues.  Otherwise the compiler
 332 ;; may chose to hold the tracking state encoded in SP.
 333 (define_attr "speculation_barrier" "true,false" (const_string "false"))
 334
 335 ;; -------------------------------------------------------------------
 336 ;; Pipeline descriptions and scheduling
 337 ;; -------------------------------------------------------------------
 338
 339 ;; Processor types.
 340 (include "aarch64-tune.md")
 341
 342 ;; Scheduling
 343 (include "../arm/cortex-a53.md")
 344 (include "../arm/cortex-a57.md")
 345 (include "../arm/exynos-m1.md")
 346 (include "falkor.md")
 347 (include "thunderx.md")
 348 (include "../arm/xgene1.md")
 349 (include "thunderx2t99.md")
 350
 351 ;; -------------------------------------------------------------------
 352 ;; Jumps and other miscellaneous insns
 353 ;; -------------------------------------------------------------------
 354
 355 (define_insn "indirect_jump"
 356   [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
 357   ""
 358   "br\\t%0"
 359   [(set_attr "type" "branch")]
 360 )
 361
 362 (define_insn "jump"
 363   [(set (pc) (label_ref (match_operand 0 "" "")))]
 364   ""
 365   "b\\t%l0"
 366   [(set_attr "type" "branch")]
 367 )
 368
 369 (define_expand "cbranch<mode>4"
 370   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 371                             [(match_operand:GPI 1 "register_operand" "")
 372                              (match_operand:GPI 2 "aarch64_plus_operand" "")])
 373                            (label_ref (match_operand 3 "" ""))
 374                            (pc)))]
 375   ""
 376   "
 377   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 378                                          operands[2]);
 379   operands[2] = const0_rtx;
 380   "
 381 )
 382
 383 (define_expand "cbranch<mode>4"
 384   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 385                             [(match_operand:GPF 1 "register_operand" "")
 386                              (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
 387                            (label_ref (match_operand 3 "" ""))
 388                            (pc)))]
 389   ""
 390   "
 391   operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
 392                                          operands[2]);
 393   operands[2] = const0_rtx;
 394   "
 395 )
 396
 397 (define_expand "cbranchcc4"
 398   [(set (pc) (if_then_else
 399               (match_operator 0 "aarch64_comparison_operator"
 400                [(match_operand 1 "cc_register" "")
 401                 (match_operand 2 "const0_operand")])
 402               (label_ref (match_operand 3 "" ""))
 403               (pc)))]
 404   ""
 405   "")
 406
 407 (define_insn "ccmp<mode>"
 408   [(set (match_operand:CC 1 "cc_register" "")
 409         (if_then_else:CC
 410           (match_operator 4 "aarch64_comparison_operator"
 411            [(match_operand 0 "cc_register" "")
 412             (const_int 0)])
 413           (compare:CC
 414             (match_operand:GPI 2 "register_operand" "r,r,r")
 415             (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
 416           (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 417   ""
 418   "@
 419    ccmp\\t%<w>2, %<w>3, %k5, %m4
 420    ccmp\\t%<w>2, %3, %k5, %m4
 421    ccmn\\t%<w>2, #%n3, %k5, %m4"
 422   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
 423 )
 424
 425 (define_insn "fccmp<mode>"
 426   [(set (match_operand:CCFP 1 "cc_register" "")
 427         (if_then_else:CCFP
 428           (match_operator 4 "aarch64_comparison_operator"
 429            [(match_operand 0 "cc_register" "")
 430             (const_int 0)])
 431           (compare:CCFP
 432             (match_operand:GPF 2 "register_operand" "w")
 433             (match_operand:GPF 3 "register_operand" "w"))
 434           (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 435   "TARGET_FLOAT"
 436   "fccmp\\t%<s>2, %<s>3, %k5, %m4"
 437   [(set_attr "type" "fccmp<s>")]
 438 )
 439
 440 (define_insn "fccmpe<mode>"
 441   [(set (match_operand:CCFPE 1 "cc_register" "")
 442          (if_then_else:CCFPE
 443           (match_operator 4 "aarch64_comparison_operator"
 444            [(match_operand 0 "cc_register" "")
 445           (const_int 0)])
 446            (compare:CCFPE
 447             (match_operand:GPF 2 "register_operand" "w")
 448             (match_operand:GPF 3 "register_operand" "w"))
 449           (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
 450   "TARGET_FLOAT"
 451   "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
 452   [(set_attr "type" "fccmp<s>")]
 453 )
 454
 455 ;; Expansion of signed mod by a power of 2 using CSNEG.
 456 ;; For x0 % n where n is a power of 2 produce:
 457 ;; negs   x1, x0
 458 ;; and    x0, x0, #(n - 1)
 459 ;; and    x1, x1, #(n - 1)
 460 ;; csneg  x0, x0, x1, mi
 461
 462 (define_expand "mod<mode>3"
 463   [(match_operand:GPI 0 "register_operand" "")
 464    (match_operand:GPI 1 "register_operand" "")
 465    (match_operand:GPI 2 "const_int_operand" "")]
 466   ""
 467   {
 468     HOST_WIDE_INT val = INTVAL (operands[2]);
 469
 470     if (val <= 0
 471        || exact_log2 (val) <= 0
 472        || !aarch64_bitmask_imm (val - 1, <MODE>mode))
 473       FAIL;
 474
 475     rtx mask = GEN_INT (val - 1);
 476
 477     /* In the special case of x0 % 2 we can do the even shorter:
 478         cmp    x0, xzr
 479         and    x0, x0, 1
 480         cneg   x0, x0, lt.  */
 481     if (val == 2)
 482       {
 483         rtx masked = gen_reg_rtx (<MODE>mode);
 484         rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
 485         emit_insn (gen_and<mode>3 (masked, operands[1], mask));
 486         rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
 487         emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
 488         DONE;
 489       }
 490
 491     rtx neg_op = gen_reg_rtx (<MODE>mode);
 492     rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
 493
 494     /* Extract the condition register and mode.  */
 495     rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
 496     rtx cc_reg = SET_DEST (cmp);
 497     rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
 498
 499     rtx masked_pos = gen_reg_rtx (<MODE>mode);
 500     emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
 501
 502     rtx masked_neg = gen_reg_rtx (<MODE>mode);
 503     emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
 504
 505     emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
 506                                        masked_neg, masked_pos));
 507     DONE;
 508   }
 509 )
 510
 511 (define_insn "condjump"
 512   [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
 513                             [(match_operand 1 "cc_register" "") (const_int 0)])
 514                            (label_ref (match_operand 2 "" ""))
 515                            (pc)))]
 516   ""
 517   {
 518     if (get_attr_length (insn) == 8)
 519       return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
 520     else
 521       return  "b%m0\\t%l2";
 522   }
 523   [(set_attr "type" "branch")
 524    (set (attr "length")
 525         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 526                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 527                       (const_int 4)
 528                       (const_int 8)))
 529    (set (attr "far_branch")
 530         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 531                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 532                       (const_int 0)
 533                       (const_int 1)))]
 534 )
 535
 536 ;; For a 24-bit immediate CST we can optimize the compare for equality
 537 ;; and branch sequence from:
 538 ;;      mov     x0, #imm1
 539 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
 540 ;;      cmp     x1, x0
 541 ;;      b<ne,eq> .Label
 542 ;; into the shorter:
 543 ;;      sub     x0, x1, #(CST & 0xfff000)
 544 ;;      subs    x0, x0, #(CST & 0x000fff)
 545 ;;      b<ne,eq> .Label
 546 (define_insn_and_split "*compare_condjump<mode>"
 547   [(set (pc) (if_then_else (EQL
 548                               (match_operand:GPI 0 "register_operand" "r")
 549                               (match_operand:GPI 1 "aarch64_imm24" "n"))
 550                            (label_ref:P (match_operand 2 "" ""))
 551                            (pc)))]
 552   "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
 553    && !aarch64_plus_operand (operands[1], <MODE>mode)
 554    && !reload_completed"
 555   "#"
 556   "&& true"
 557   [(const_int 0)]
 558   {
 559     HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
 560     HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
 561     rtx tmp = gen_reg_rtx (<MODE>mode);
 562     emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
 563     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
 564     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
 565     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
 566     emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
 567     DONE;
 568   }
 569 )
 570
 571 (define_expand "casesi"
 572   [(match_operand:SI 0 "register_operand" "")   ; Index
 573    (match_operand:SI 1 "const_int_operand" "")  ; Lower bound
 574    (match_operand:SI 2 "const_int_operand" "")  ; Total range
 575    (match_operand:DI 3 "" "")                   ; Table label
 576    (match_operand:DI 4 "" "")]                  ; Out of range label
 577   ""
 578   {
 579     if (operands[1] != const0_rtx)
 580       {
 581         rtx reg = gen_reg_rtx (SImode);
 582
 583         /* Canonical RTL says that if you have:
 584
 585            (minus (X) (CONST))
 586
 587            then this should be emitted as:
 588
 589            (plus (X) (-CONST))
 590
 591            The use of trunc_int_for_mode ensures that the resulting
 592            constant can be represented in SImode, this is important
 593            for the corner case where operand[1] is INT_MIN.  */
 594
 595         operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
 596
 597         if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
 598               (operands[1], SImode))
 599           operands[1] = force_reg (SImode, operands[1]);
 600         emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
 601         operands[0] = reg;
 602       }
 603
 604     if (!aarch64_plus_operand (operands[2], SImode))
 605       operands[2] = force_reg (SImode, operands[2]);
 606     emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
 607                                                  const0_rtx),
 608                                     operands[0], operands[2], operands[4]));
 609
 610     operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
 611     emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
 612                                          operands[3]));
 613     DONE;
 614   }
 615 )
 616
 617 (define_insn "casesi_dispatch"
 618   [(parallel
 619     [(set (pc)
 620           (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
 621                            (match_operand:SI 1 "register_operand" "r")]
 622                         UNSPEC_CASESI)))
 623      (clobber (reg:CC CC_REGNUM))
 624      (clobber (match_scratch:DI 3 "=r"))
 625      (clobber (match_scratch:DI 4 "=r"))
 626      (use (label_ref (match_operand 2 "" "")))])]
 627   ""
 628   "*
 629   return aarch64_output_casesi (operands);
 630   "
 631   [(set_attr "length" "16")
 632    (set_attr "type" "branch")]
 633 )
 634
 635 (define_insn "nop"
 636   [(unspec[(const_int 0)] UNSPEC_NOP)]
 637   ""
 638   "nop"
 639   [(set_attr "type" "no_insn")]
 640 )
 641
 642 (define_insn "prefetch"
 643   [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
 644             (match_operand:QI 1 "const_int_operand" "")
 645             (match_operand:QI 2 "const_int_operand" ""))]
 646   ""
 647   {
 648     const char * pftype[2][4] =
 649     {
 650       {"prfm\\tPLDL1STRM, %0",
 651        "prfm\\tPLDL3KEEP, %0",
 652        "prfm\\tPLDL2KEEP, %0",
 653        "prfm\\tPLDL1KEEP, %0"},
 654       {"prfm\\tPSTL1STRM, %0",
 655        "prfm\\tPSTL3KEEP, %0",
 656        "prfm\\tPSTL2KEEP, %0",
 657        "prfm\\tPSTL1KEEP, %0"},
 658     };
 659
 660     int locality = INTVAL (operands[2]);
 661
 662     gcc_assert (IN_RANGE (locality, 0, 3));
 663
 664     /* PRFM accepts the same addresses as a 64-bit LDR so wrap
 665        the address into a DImode MEM so that aarch64_print_operand knows
 666        how to print it.  */
 667     operands[0] = gen_rtx_MEM (DImode, operands[0]);
 668     return pftype[INTVAL(operands[1])][locality];
 669   }
 670   [(set_attr "type" "load_4")]
 671 )
 672
 673 (define_insn "trap"
 674   [(trap_if (const_int 1) (const_int 8))]
 675   ""
 676   "brk #1000"
 677   [(set_attr "type" "trap")])
 678
 679 (define_expand "prologue"
 680   [(clobber (const_int 0))]
 681   ""
 682   "
 683   aarch64_expand_prologue ();
 684   DONE;
 685   "
 686 )
 687
 688 (define_expand "epilogue"
 689   [(clobber (const_int 0))]
 690   ""
 691   "
 692   aarch64_expand_epilogue (false);
 693   DONE;
 694   "
 695 )
 696
 697 (define_expand "sibcall_epilogue"
 698   [(clobber (const_int 0))]
 699   ""
 700   "
 701   aarch64_expand_epilogue (true);
 702   DONE;
 703   "
 704 )
 705
 706 (define_insn "*do_return"
 707   [(return)]
 708   ""
 709   {
 710     if (aarch64_return_address_signing_enabled ()
 711         && TARGET_ARMV8_3
 712         && !crtl->calls_eh_return)
 713       return "retaa";
 714
 715     return "ret";
 716   }
 717   [(set_attr "type" "branch")]
 718 )
 719
 720 (define_expand "return"
 721   [(simple_return)]
 722   "aarch64_use_return_insn_p ()"
 723   ""
 724 )
 725
 726 (define_insn "simple_return"
 727   [(simple_return)]
 728   ""
 729   "ret"
 730   [(set_attr "type" "branch")]
 731 )
 732
 733 (define_insn "*cb<optab><mode>1"
 734   [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
 735                                 (const_int 0))
 736                            (label_ref (match_operand 1 "" ""))
 737                            (pc)))]
 738   "!aarch64_track_speculation"
 739   {
 740     if (get_attr_length (insn) == 8)
 741       return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
 742     else
 743       return "<cbz>\\t%<w>0, %l1";
 744   }
 745   [(set_attr "type" "branch")
 746    (set (attr "length")
 747         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 748                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 749                       (const_int 4)
 750                       (const_int 8)))
 751    (set (attr "far_branch")
 752         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 753                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 754                       (const_int 0)
 755                       (const_int 1)))]
 756 )
 757
 758 (define_insn "*tb<optab><mode>1"
 759   [(set (pc) (if_then_else
 760               (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
 761                                     (const_int 1)
 762                                     (match_operand 1
 763                                       "aarch64_simd_shift_imm_<mode>" "n"))
 764                    (const_int 0))
 765              (label_ref (match_operand 2 "" ""))
 766              (pc)))
 767    (clobber (reg:CC CC_REGNUM))]
 768   "!aarch64_track_speculation"
 769   {
 770     if (get_attr_length (insn) == 8)
 771       {
 772         if (get_attr_far_branch (insn) == 1)
 773           return aarch64_gen_far_branch (operands, 2, "Ltb",
 774                                          "<inv_tb>\\t%<w>0, %1, ");
 775         else
 776           {
 777             operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
 778             return "tst\t%<w>0, %1\;<bcond>\t%l2";
 779           }
 780       }
 781     else
 782       return "<tbz>\t%<w>0, %1, %l2";
 783   }
 784   [(set_attr "type" "branch")
 785    (set (attr "length")
 786         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
 787                            (lt (minus (match_dup 2) (pc)) (const_int 32764)))
 788                       (const_int 4)
 789                       (const_int 8)))
 790    (set (attr "far_branch")
 791         (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
 792                            (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
 793                       (const_int 0)
 794                       (const_int 1)))]
 795
 796 )
 797
 798 (define_insn "*cb<optab><mode>1"
 799   [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
 800                                  (const_int 0))
 801                            (label_ref (match_operand 1 "" ""))
 802                            (pc)))
 803    (clobber (reg:CC CC_REGNUM))]
 804   "!aarch64_track_speculation"
 805   {
 806     if (get_attr_length (insn) == 8)
 807       {
 808         if (get_attr_far_branch (insn) == 1)
 809           return aarch64_gen_far_branch (operands, 1, "Ltb",
 810                                          "<inv_tb>\\t%<w>0, <sizem1>, ");
 811         else
 812           {
 813             char buf[64];
 814             uint64_t val = ((uint64_t) 1)
 815                 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
 816             sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
 817             output_asm_insn (buf, operands);
 818             return "<bcond>\t%l1";
 819           }
 820       }
 821     else
 822       return "<tbz>\t%<w>0, <sizem1>, %l1";
 823   }
 824   [(set_attr "type" "branch")
 825    (set (attr "length")
 826         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
 827                            (lt (minus (match_dup 1) (pc)) (const_int 32764)))
 828                       (const_int 4)
 829                       (const_int 8)))
 830    (set (attr "far_branch")
 831         (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
 832                            (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
 833                       (const_int 0)
 834                       (const_int 1)))]
 835 )
 836
 837 ;; -------------------------------------------------------------------
 838 ;; Subroutine calls and sibcalls
 839 ;; -------------------------------------------------------------------
 840
 841 (define_expand "call"
 842   [(parallel [(call (match_operand 0 "memory_operand" "")
 843                     (match_operand 1 "general_operand" ""))
 844               (use (match_operand 2 "" ""))
 845               (clobber (reg:DI LR_REGNUM))])]
 846   ""
 847   "
 848   {
 849     aarch64_expand_call (NULL_RTX, operands[0], false);
 850     DONE;
 851   }"
 852 )
 853
 854 (define_insn "*call_insn"
 855   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
 856          (match_operand 1 "" ""))
 857    (clobber (reg:DI LR_REGNUM))]
 858   ""
 859   "@
 860   blr\\t%0
 861   bl\\t%c0"
 862   [(set_attr "type" "call, call")]
 863 )
 864
 865 (define_expand "call_value"
 866   [(parallel [(set (match_operand 0 "" "")
 867                    (call (match_operand 1 "memory_operand" "")
 868                          (match_operand 2 "general_operand" "")))
 869               (use (match_operand 3 "" ""))
 870               (clobber (reg:DI LR_REGNUM))])]
 871   ""
 872   "
 873   {
 874     aarch64_expand_call (operands[0], operands[1], false);
 875     DONE;
 876   }"
 877 )
 878
 879 (define_insn "*call_value_insn"
 880   [(set (match_operand 0 "" "")
 881         (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
 882                       (match_operand 2 "" "")))
 883    (clobber (reg:DI LR_REGNUM))]
 884   ""
 885   "@
 886   blr\\t%1
 887   bl\\t%c1"
 888   [(set_attr "type" "call, call")]
 889 )
 890
 891 (define_expand "sibcall"
 892   [(parallel [(call (match_operand 0 "memory_operand" "")
 893                     (match_operand 1 "general_operand" ""))
 894               (return)
 895               (use (match_operand 2 "" ""))])]
 896   ""
 897   {
 898     aarch64_expand_call (NULL_RTX, operands[0], true);
 899     DONE;
 900   }
 901 )
 902
 903 (define_expand "sibcall_value"
 904   [(parallel [(set (match_operand 0 "" "")
 905                    (call (match_operand 1 "memory_operand" "")
 906                          (match_operand 2 "general_operand" "")))
 907               (return)
 908               (use (match_operand 3 "" ""))])]
 909   ""
 910   {
 911     aarch64_expand_call (operands[0], operands[1], true);
 912     DONE;
 913   }
 914 )
 915
 916 (define_insn "*sibcall_insn"
 917   [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
 918          (match_operand 1 "" ""))
 919    (return)]
 920   "SIBLING_CALL_P (insn)"
 921   "@
 922    br\\t%0
 923    b\\t%c0"
 924   [(set_attr "type" "branch, branch")]
 925 )
 926
 927 (define_insn "*sibcall_value_insn"
 928   [(set (match_operand 0 "" "")
 929         (call (mem:DI
 930                 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
 931               (match_operand 2 "" "")))
 932    (return)]
 933   "SIBLING_CALL_P (insn)"
 934   "@
 935    br\\t%1
 936    b\\t%c1"
 937   [(set_attr "type" "branch, branch")]
 938 )
 939
 940 ;; Call subroutine returning any type.
 941
 942 (define_expand "untyped_call"
 943   [(parallel [(call (match_operand 0 "")
 944                     (const_int 0))
 945               (match_operand 1 "")
 946               (match_operand 2 "")])]
 947   ""
 948 {
 949   int i;
 950
 951   emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
 952
 953   for (i = 0; i < XVECLEN (operands[2], 0); i++)
 954     {
 955       rtx set = XVECEXP (operands[2], 0, i);
 956       emit_move_insn (SET_DEST (set), SET_SRC (set));
 957     }
 958
 959   /* The optimizer does not know that the call sets the function value
 960      registers we stored in the result block.  We avoid problems by
 961      claiming that all hard registers are used and clobbered at this
 962      point.  */
 963   emit_insn (gen_blockage ());
 964   DONE;
 965 })
 966
 967 ;; -------------------------------------------------------------------
 968 ;; Moves
 969 ;; -------------------------------------------------------------------
 970
 971 (define_expand "mov<mode>"
 972   [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
 973         (match_operand:SHORT 1 "general_operand" ""))]
 974   ""
 975   "
 976     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
 977       operands[1] = force_reg (<MODE>mode, operands[1]);
 978
 979     if (GET_CODE (operands[1]) == CONST_POLY_INT)
 980       {
 981         aarch64_expand_mov_immediate (operands[0], operands[1]);
 982         DONE;
 983       }
 984   "
 985 )
 986
 987 (define_insn "*mov<mode>_aarch64"
 988   [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r,    w,r  ,r,w, m,m,r,w,w")
 989         (match_operand:SHORT 1 "aarch64_mov_operand"  " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
 990   "(register_operand (operands[0], <MODE>mode)
 991     || aarch64_reg_or_zero (operands[1], <MODE>mode))"
 992 {
 993    switch (which_alternative)
 994      {
 995      case 0:
 996        return "mov\t%w0, %w1";
 997      case 1:
 998        return "mov\t%w0, %1";
 999      case 2:
1000        return aarch64_output_scalar_simd_mov_immediate (operands[1],
1001                                                         <MODE>mode);
1002      case 3:
1003        return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1004      case 4:
1005        return "ldr<size>\t%w0, %1";
1006      case 5:
1007        return "ldr\t%<size>0, %1";
1008      case 6:
1009        return "str<size>\t%w1, %0";
1010      case 7:
1011        return "str\t%<size>1, %0";
1012      case 8:
1013        return "umov\t%w0, %1.<v>[0]";
1014      case 9:
1015        return "dup\t%0.<Vallxd>, %w1";
1016      case 10:
1017        return "dup\t%<Vetype>0, %1.<v>[0]";
1018      default:
1019        gcc_unreachable ();
1020      }
1021 }
1022   ;; The "mov_imm" type for CNT is just a placeholder.
1023   [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1024                      store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1025    (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1026 )
1027
1028 (define_expand "mov<mode>"
1029   [(set (match_operand:GPI 0 "nonimmediate_operand" "")
1030         (match_operand:GPI 1 "general_operand" ""))]
1031   ""
1032   "
1033     if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1034         && <MODE>mode == DImode
1035         && aarch64_split_dimode_const_store (operands[0], operands[1]))
1036       DONE;
1037
1038     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1039       operands[1] = force_reg (<MODE>mode, operands[1]);
1040
1041     /* FIXME: RR we still need to fix up what we are doing with
1042        symbol_refs and other types of constants.  */
1043     if (CONSTANT_P (operands[1])
1044         && !CONST_INT_P (operands[1]))
1045      {
1046        aarch64_expand_mov_immediate (operands[0], operands[1]);
1047        DONE;
1048      }
1049   "
1050 )
1051
1052 (define_insn_and_split "*movsi_aarch64"
1053   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m,  r,  r, w,r,w, w")
1054         (match_operand:SI 1 "aarch64_mov_operand"  " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1055   "(register_operand (operands[0], SImode)
1056     || aarch64_reg_or_zero (operands[1], SImode))"
1057   "@
1058    mov\\t%w0, %w1
1059    mov\\t%w0, %w1
1060    mov\\t%w0, %w1
1061    mov\\t%w0, %1
1062    #
1063    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1064    ldr\\t%w0, %1
1065    ldr\\t%s0, %1
1066    str\\t%w1, %0
1067    str\\t%s1, %0
1068    adr\\t%x0, %c1
1069    adrp\\t%x0, %A1
1070    fmov\\t%s0, %w1
1071    fmov\\t%w0, %s1
1072    fmov\\t%s0, %s1
1073    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1074   "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1075     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1076    [(const_int 0)]
1077    "{
1078        aarch64_expand_mov_immediate (operands[0], operands[1]);
1079        DONE;
1080     }"
1081   ;; The "mov_imm" type for CNT is just a placeholder.
1082   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1083                     load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1084    (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1085 )
1086
1087 (define_insn_and_split "*movdi_aarch64"
1088   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m,  r,  r, w,r,w, w")
1089         (match_operand:DI 1 "aarch64_mov_operand"  " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1090   "(register_operand (operands[0], DImode)
1091     || aarch64_reg_or_zero (operands[1], DImode))"
1092   "@
1093    mov\\t%x0, %x1
1094    mov\\t%0, %x1
1095    mov\\t%x0, %1
1096    mov\\t%x0, %1
1097    mov\\t%w0, %1
1098    #
1099    * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1100    ldr\\t%x0, %1
1101    ldr\\t%d0, %1
1102    str\\t%x1, %0
1103    str\\t%d1, %0
1104    adr\\t%x0, %c1
1105    adrp\\t%x0, %A1
1106    fmov\\t%d0, %x1
1107    fmov\\t%x0, %d1
1108    fmov\\t%d0, %d1
1109    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1110    "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1111     && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1112    [(const_int 0)]
1113    "{
1114        aarch64_expand_mov_immediate (operands[0], operands[1]);
1115        DONE;
1116     }"
1117   ;; The "mov_imm" type for CNTD is just a placeholder.
1118   [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1119                      load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1120                      neon_move")
1121    (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1122 )
1123
1124 (define_insn "insv_imm<mode>"
1125   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1126                           (const_int 16)
1127                           (match_operand:GPI 1 "const_int_operand" "n"))
1128         (match_operand:GPI 2 "const_int_operand" "n"))]
1129   "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1130    && UINTVAL (operands[1]) % 16 == 0"
1131   "movk\\t%<w>0, %X2, lsl %1"
1132   [(set_attr "type" "mov_imm")]
1133 )
1134
1135 (define_expand "movti"
1136   [(set (match_operand:TI 0 "nonimmediate_operand" "")
1137         (match_operand:TI 1 "general_operand" ""))]
1138   ""
1139   "
1140     if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1141       operands[1] = force_reg (TImode, operands[1]);
1142
1143     if (GET_CODE (operands[1]) == CONST_POLY_INT)
1144       {
1145         emit_move_insn (gen_lowpart (DImode, operands[0]),
1146                         gen_lowpart (DImode, operands[1]));
1147         emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1148         DONE;
1149       }
1150   "
1151 )
1152
1153 (define_insn "*movti_aarch64"
1154   [(set (match_operand:TI 0
1155          "nonimmediate_operand"  "=   r,w, r,w,r,m,m,w,m")
1156         (match_operand:TI 1
1157          "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1158   "(register_operand (operands[0], TImode)
1159     || aarch64_reg_or_zero (operands[1], TImode))"
1160   "@
1161    #
1162    #
1163    #
1164    mov\\t%0.16b, %1.16b
1165    ldp\\t%0, %H0, %1
1166    stp\\t%1, %H1, %0
1167    stp\\txzr, xzr, %0
1168    ldr\\t%q0, %1
1169    str\\t%q1, %0"
1170   [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1171                              load_16,store_16,store_16,\
1172                              load_16,store_16")
1173    (set_attr "length" "8,8,8,4,4,4,4,4,4")
1174    (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1175 )
1176
1177 ;; Split a TImode register-register or register-immediate move into
1178 ;; its component DImode pieces, taking care to handle overlapping
1179 ;; source and dest registers.
1180 (define_split
1181    [(set (match_operand:TI 0 "register_operand" "")
1182          (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1183   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1184   [(const_int 0)]
1185 {
1186   aarch64_split_128bit_move (operands[0], operands[1]);
1187   DONE;
1188 })
1189
1190 (define_expand "mov<mode>"
1191   [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1192         (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1193   ""
1194   {
1195     if (!TARGET_FLOAT)
1196       {
1197         aarch64_err_no_fpadvsimd (<MODE>mode);
1198         FAIL;
1199       }
1200
1201     if (GET_CODE (operands[0]) == MEM
1202         && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1203               && aarch64_float_const_zero_rtx_p (operands[1])))
1204       operands[1] = force_reg (<MODE>mode, operands[1]);
1205   }
1206 )
1207
1208 (define_insn "*movhf_aarch64"
1209   [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w  , w,?r,w,w  ,w  ,w,m,r,m ,r")
1210         (match_operand:HF 1 "general_operand"      "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1211   "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1212     || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1213   "@
1214    movi\\t%0.4h, #0
1215    fmov\\t%h0, %w1
1216    dup\\t%w0.4h, %w1
1217    umov\\t%w0, %1.h[0]
1218    mov\\t%0.h[0], %1.h[0]
1219    fmov\\t%h0, %1
1220    * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1221    ldr\\t%h0, %1
1222    str\\t%h1, %0
1223    ldrh\\t%w0, %1
1224    strh\\t%w1, %0
1225    mov\\t%w0, %w1"
1226   [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1227                      neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1228    (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1229 )
1230
1231 (define_insn "*movsf_aarch64"
1232   [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1233         (match_operand:SF 1 "general_operand"      "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1234   "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1235     || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1236   "@
1237    movi\\t%0.2s, #0
1238    fmov\\t%s0, %w1
1239    fmov\\t%w0, %s1
1240    fmov\\t%s0, %s1
1241    fmov\\t%s0, %1
1242    * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1243    ldr\\t%s0, %1
1244    str\\t%s1, %0
1245    ldr\\t%w0, %1
1246    str\\t%w1, %0
1247    mov\\t%w0, %w1
1248    mov\\t%w0, %1"
1249   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1250                      f_loads,f_stores,load_4,store_4,mov_reg,\
1251                      fconsts")
1252    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1253 )
1254
1255 (define_insn "*movdf_aarch64"
1256   [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w  ,?r,w,w  ,w  ,w,m,r,m ,r,r")
1257         (match_operand:DF 1 "general_operand"      "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1258   "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1259     || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1260   "@
1261    movi\\t%d0, #0
1262    fmov\\t%d0, %x1
1263    fmov\\t%x0, %d1
1264    fmov\\t%d0, %d1
1265    fmov\\t%d0, %1
1266    * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1267    ldr\\t%d0, %1
1268    str\\t%d1, %0
1269    ldr\\t%x0, %1
1270    str\\t%x1, %0
1271    mov\\t%x0, %x1
1272    mov\\t%x0, %1"
1273   [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1274                      f_loadd,f_stored,load_8,store_8,mov_reg,\
1275                      fconstd")
1276    (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1277 )
1278
1279 (define_split
1280   [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1281         (match_operand:GPF_HF 1 "general_operand"))]
1282   "can_create_pseudo_p ()
1283    && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1284    && !aarch64_float_const_representable_p (operands[1])
1285    &&  aarch64_float_const_rtx_p (operands[1])"
1286   [(const_int 0)]
1287   {
1288     unsigned HOST_WIDE_INT ival;
1289     if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1290       FAIL;
1291
1292     rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1293     emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1294     emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1295     DONE;
1296   }
1297 )
1298
1299 (define_insn "*movtf_aarch64"
1300   [(set (match_operand:TF 0
1301          "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1302         (match_operand:TF 1
1303          "general_operand"      " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1304   "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1305     || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1306   "@
1307    mov\\t%0.16b, %1.16b
1308    #
1309    #
1310    #
1311    movi\\t%0.2d, #0
1312    fmov\\t%s0, wzr
1313    ldr\\t%q0, %1
1314    str\\t%q1, %0
1315    ldp\\t%0, %H0, %1
1316    stp\\t%1, %H1, %0
1317    stp\\txzr, xzr, %0"
1318   [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1319                      f_loadd,f_stored,load_16,store_16,store_16")
1320    (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1321    (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1322 )
1323
1324 (define_split
1325    [(set (match_operand:TF 0 "register_operand" "")
1326          (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1327   "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1328   [(const_int 0)]
1329   {
1330     aarch64_split_128bit_move (operands[0], operands[1]);
1331     DONE;
1332   }
1333 )
1334
1335 ;; 0 is dst
1336 ;; 1 is src
1337 ;; 2 is size of move in bytes
1338 ;; 3 is alignment
1339
1340 (define_expand "movmemdi"
1341   [(match_operand:BLK 0 "memory_operand")
1342    (match_operand:BLK 1 "memory_operand")
1343    (match_operand:DI 2 "immediate_operand")
1344    (match_operand:DI 3 "immediate_operand")]
1345    "!STRICT_ALIGNMENT"
1346 {
1347   if (aarch64_expand_movmem (operands))
1348     DONE;
1349   FAIL;
1350 }
1351 )
1352
1353 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1354 ;; fairly lax checking on the second memory operation.
1355 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1356   [(set (match_operand:SX 0 "register_operand" "=r,w")
1357         (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1358    (set (match_operand:SX2 2 "register_operand" "=r,w")
1359         (match_operand:SX2 3 "memory_operand" "m,m"))]
1360    "rtx_equal_p (XEXP (operands[3], 0),
1361                  plus_constant (Pmode,
1362                                 XEXP (operands[1], 0),
1363                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1364   "@
1365    ldp\\t%w0, %w2, %1
1366    ldp\\t%s0, %s2, %1"
1367   [(set_attr "type" "load_8,neon_load1_2reg")
1368    (set_attr "arch" "*,fp")]
1369 )
1370
1371 ;; Storing different modes that can still be merged
1372 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1373   [(set (match_operand:DX 0 "register_operand" "=r,w")
1374         (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1375    (set (match_operand:DX2 2 "register_operand" "=r,w")
1376         (match_operand:DX2 3 "memory_operand" "m,m"))]
1377    "rtx_equal_p (XEXP (operands[3], 0),
1378                  plus_constant (Pmode,
1379                                 XEXP (operands[1], 0),
1380                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1381   "@
1382    ldp\\t%x0, %x2, %1
1383    ldp\\t%d0, %d2, %1"
1384   [(set_attr "type" "load_16,neon_load1_2reg")
1385    (set_attr "arch" "*,fp")]
1386 )
1387
1388 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1389 ;; fairly lax checking on the second memory operation.
1390 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1391   [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1392         (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1393    (set (match_operand:SX2 2 "memory_operand" "=m,m")
1394         (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1395    "rtx_equal_p (XEXP (operands[2], 0),
1396                  plus_constant (Pmode,
1397                                 XEXP (operands[0], 0),
1398                                 GET_MODE_SIZE (<SX:MODE>mode)))"
1399   "@
1400    stp\\t%w1, %w3, %0
1401    stp\\t%s1, %s3, %0"
1402   [(set_attr "type" "store_8,neon_store1_2reg")
1403    (set_attr "arch" "*,fp")]
1404 )
1405
1406 ;; Storing different modes that can still be merged
1407 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1408   [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1409         (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1410    (set (match_operand:DX2 2 "memory_operand" "=m,m")
1411         (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1412    "rtx_equal_p (XEXP (operands[2], 0),
1413                  plus_constant (Pmode,
1414                                 XEXP (operands[0], 0),
1415                                 GET_MODE_SIZE (<DX:MODE>mode)))"
1416   "@
1417    stp\\t%x1, %x3, %0
1418    stp\\t%d1, %d3, %0"
1419   [(set_attr "type" "store_16,neon_store1_2reg")
1420    (set_attr "arch" "*,fp")]
1421 )
1422
1423 ;; Load pair with post-index writeback.  This is primarily used in function
1424 ;; epilogues.
1425 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1426   [(parallel
1427     [(set (match_operand:P 0 "register_operand" "=k")
1428           (plus:P (match_operand:P 1 "register_operand" "0")
1429                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1430      (set (match_operand:GPI 2 "register_operand" "=r")
1431           (mem:GPI (match_dup 1)))
1432      (set (match_operand:GPI 3 "register_operand" "=r")
1433           (mem:GPI (plus:P (match_dup 1)
1434                    (match_operand:P 5 "const_int_operand" "n"))))])]
1435   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1436   "ldp\\t%<w>2, %<w>3, [%1], %4"
1437   [(set_attr "type" "load_<ldpstp_sz>")]
1438 )
1439
1440 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1441   [(parallel
1442     [(set (match_operand:P 0 "register_operand" "=k")
1443           (plus:P (match_operand:P 1 "register_operand" "0")
1444                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1445      (set (match_operand:GPF 2 "register_operand" "=w")
1446           (mem:GPF (match_dup 1)))
1447      (set (match_operand:GPF 3 "register_operand" "=w")
1448           (mem:GPF (plus:P (match_dup 1)
1449                    (match_operand:P 5 "const_int_operand" "n"))))])]
1450   "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1451   "ldp\\t%<w>2, %<w>3, [%1], %4"
1452   [(set_attr "type" "neon_load1_2reg")]
1453 )
1454
1455 ;; Store pair with pre-index writeback.  This is primarily used in function
1456 ;; prologues.
1457 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1458   [(parallel
1459     [(set (match_operand:P 0 "register_operand" "=&k")
1460           (plus:P (match_operand:P 1 "register_operand" "0")
1461                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1462      (set (mem:GPI (plus:P (match_dup 0)
1463                    (match_dup 4)))
1464           (match_operand:GPI 2 "register_operand" "r"))
1465      (set (mem:GPI (plus:P (match_dup 0)
1466                    (match_operand:P 5 "const_int_operand" "n")))
1467           (match_operand:GPI 3 "register_operand" "r"))])]
1468   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1469   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1470   [(set_attr "type" "store_<ldpstp_sz>")]
1471 )
1472
1473 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1474   [(parallel
1475     [(set (match_operand:P 0 "register_operand" "=&k")
1476           (plus:P (match_operand:P 1 "register_operand" "0")
1477                   (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1478      (set (mem:GPF (plus:P (match_dup 0)
1479                    (match_dup 4)))
1480           (match_operand:GPF 2 "register_operand" "w"))
1481      (set (mem:GPF (plus:P (match_dup 0)
1482                    (match_operand:P 5 "const_int_operand" "n")))
1483           (match_operand:GPF 3 "register_operand" "w"))])]
1484   "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1485   "stp\\t%<w>2, %<w>3, [%0, %4]!"
1486   [(set_attr "type" "neon_store1_2reg<q>")]
1487 )
1488
1489 ;; -------------------------------------------------------------------
1490 ;; Sign/Zero extension
1491 ;; -------------------------------------------------------------------
1492
1493 (define_expand "<optab>sidi2"
1494   [(set (match_operand:DI 0 "register_operand")
1495         (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1496   ""
1497 )
1498
1499 (define_insn "*extendsidi2_aarch64"
1500   [(set (match_operand:DI 0 "register_operand" "=r,r")
1501         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1502   ""
1503   "@
1504    sxtw\t%0, %w1
1505    ldrsw\t%0, %1"
1506   [(set_attr "type" "extend,load_4")]
1507 )
1508
1509 (define_insn "*load_pair_extendsidi2_aarch64"
1510   [(set (match_operand:DI 0 "register_operand" "=r")
1511         (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1512    (set (match_operand:DI 2 "register_operand" "=r")
1513         (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1514   "rtx_equal_p (XEXP (operands[3], 0),
1515                 plus_constant (Pmode,
1516                                XEXP (operands[1], 0),
1517                                GET_MODE_SIZE (SImode)))"
1518   "ldpsw\\t%0, %2, %1"
1519   [(set_attr "type" "load_8")]
1520 )
1521
1522 (define_insn "*zero_extendsidi2_aarch64"
1523   [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w")
1524         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))]
1525   ""
1526   "@
1527    uxtw\t%0, %w1
1528    ldr\t%w0, %1
1529    fmov\t%s0, %w1
1530    ldr\t%s0, %1
1531    fmov\t%w0, %s1
1532    fmov\t%s0, %s1"
1533   [(set_attr "type" "extend,load_4,f_mcr,f_loads,f_mrc,fmov")
1534    (set_attr "arch" "*,*,fp,fp,fp,fp")]
1535 )
1536
1537 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1538   [(set (match_operand:DI 0 "register_operand" "=r,w")
1539         (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump")))
1540    (set (match_operand:DI 2 "register_operand" "=r,w")
1541         (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))]
1542   "rtx_equal_p (XEXP (operands[3], 0),
1543                 plus_constant (Pmode,
1544                                XEXP (operands[1], 0),
1545                                GET_MODE_SIZE (SImode)))"
1546   "@
1547    ldp\t%w0, %w2, %1
1548    ldp\t%s0, %s2, %1"
1549   [(set_attr "type" "load_8,neon_load1_2reg")
1550    (set_attr "arch" "*,fp")]
1551 )
1552
1553 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1554   [(set (match_operand:GPI 0 "register_operand")
1555         (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1556   ""
1557 )
1558
1559 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1560   [(set (match_operand:GPI 0 "register_operand" "=r,r")
1561         (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1562   ""
1563   "@
1564    sxt<SHORT:size>\t%<GPI:w>0, %w1
1565    ldrs<SHORT:size>\t%<GPI:w>0, %1"
1566   [(set_attr "type" "extend,load_4")]
1567 )
1568
1569 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1570   [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1571         (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1572   ""
1573   "@
1574    and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1575    ldr<SHORT:size>\t%w0, %1
1576    ldr\t%<SHORT:size>0, %1"
1577   [(set_attr "type" "logic_imm,load_4,f_loads")
1578    (set_attr "arch" "*,*,fp")]
1579 )
1580
1581 (define_expand "<optab>qihi2"
1582   [(set (match_operand:HI 0 "register_operand")
1583         (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1584   ""
1585 )
1586
1587 (define_insn "*extendqihi2_aarch64"
1588   [(set (match_operand:HI 0 "register_operand" "=r,r")
1589         (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1590   ""
1591   "@
1592    sxtb\t%w0, %w1
1593    ldrsb\t%w0, %1"
1594   [(set_attr "type" "extend,load_4")]
1595 )
1596
1597 (define_insn "*zero_extendqihi2_aarch64"
1598   [(set (match_operand:HI 0 "register_operand" "=r,r")
1599         (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1600   ""
1601   "@
1602    and\t%w0, %w1, 255
1603    ldrb\t%w0, %1"
1604   [(set_attr "type" "logic_imm,load_4")]
1605 )
1606
1607 ;; -------------------------------------------------------------------
1608 ;; Simple arithmetic
1609 ;; -------------------------------------------------------------------
1610
1611 (define_expand "add<mode>3"
1612   [(set
1613     (match_operand:GPI 0 "register_operand" "")
1614     (plus:GPI (match_operand:GPI 1 "register_operand" "")
1615               (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1616   ""
1617 {
1618   /* If operands[1] is a subreg extract the inner RTX.  */
1619   rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1620
1621   /* If the constant is too large for a single instruction and isn't frame
1622      based, split off the immediate so it is available for CSE.  */
1623   if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1624       && can_create_pseudo_p ()
1625       && (!REG_P (op1)
1626          || !REGNO_PTR_FRAME_P (REGNO (op1))))
1627     operands[2] = force_reg (<MODE>mode, operands[2]);
1628   /* Expand polynomial additions now if the destination is the stack
1629      pointer, since we don't want to use that as a temporary.  */
1630   else if (operands[0] == stack_pointer_rtx
1631            && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1632     {
1633       aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1634                                 operands[2], NULL_RTX, NULL_RTX);
1635       DONE;
1636     }
1637 })
1638
1639 (define_insn "*add<mode>3_aarch64"
1640   [(set
1641     (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1642     (plus:GPI
1643      (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1644      (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1645   ""
1646   "@
1647   add\\t%<w>0, %<w>1, %2
1648   add\\t%<w>0, %<w>1, %<w>2
1649   add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1650   sub\\t%<w>0, %<w>1, #%n2
1651   #
1652   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1653   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1654   [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1655    (set_attr "arch" "*,*,simd,*,*,*")]
1656 )
1657
1658 ;; zero_extend version of above
1659 (define_insn "*addsi3_aarch64_uxtw"
1660   [(set
1661     (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1662     (zero_extend:DI
1663      (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1664               (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1665   ""
1666   "@
1667   add\\t%w0, %w1, %2
1668   add\\t%w0, %w1, %w2
1669   sub\\t%w0, %w1, #%n2
1670   #"
1671   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1672 )
1673
1674 ;; If there's a free register, and we can load the constant with a
1675 ;; single instruction, do so.  This has a chance to improve scheduling.
1676 (define_peephole2
1677   [(match_scratch:GPI 3 "r")
1678    (set (match_operand:GPI 0 "register_operand")
1679         (plus:GPI
1680           (match_operand:GPI 1 "register_operand")
1681           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1682   "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1683   [(set (match_dup 3) (match_dup 2))
1684    (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1685 )
1686
1687 (define_peephole2
1688   [(match_scratch:SI 3 "r")
1689    (set (match_operand:DI 0 "register_operand")
1690         (zero_extend:DI
1691           (plus:SI
1692             (match_operand:SI 1 "register_operand")
1693             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1694   "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1695   [(set (match_dup 3) (match_dup 2))
1696    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1697 )
1698
1699 ;; After peephole2 has had a chance to run, split any remaining long
1700 ;; additions into two add immediates.
1701 (define_split
1702   [(set (match_operand:GPI 0 "register_operand")
1703         (plus:GPI
1704           (match_operand:GPI 1 "register_operand")
1705           (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1706   "epilogue_completed"
1707   [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1708    (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1709   {
1710     HOST_WIDE_INT i = INTVAL (operands[2]);
1711     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1712     operands[3] = GEN_INT (i - s);
1713     operands[4] = GEN_INT (s);
1714   }
1715 )
1716
1717 ;; Match addition of polynomial offsets that require one temporary, for which
1718 ;; we can use the early-clobbered destination register.  This is a separate
1719 ;; pattern so that the early clobber doesn't affect register allocation
1720 ;; for other forms of addition.  However, we still need to provide an
1721 ;; all-register alternative, in case the offset goes out of range after
1722 ;; elimination.  For completeness we might as well provide all GPR-based
1723 ;; alternatives from the main pattern.
1724 ;;
1725 ;; We don't have a pattern for additions requiring two temporaries since at
1726 ;; present LRA doesn't allow new scratches to be added during elimination.
1727 ;; Such offsets should be rare anyway.
1728 ;;
1729 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1730 ;; here would go away.  We could just handle all polynomial constants in
1731 ;; this pattern.
1732 (define_insn_and_split "*add<mode>3_poly_1"
1733   [(set
1734     (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1735     (plus:GPI
1736      (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1737      (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1738   "TARGET_SVE && operands[0] != stack_pointer_rtx"
1739   "@
1740   add\\t%<w>0, %<w>1, %2
1741   add\\t%<w>0, %<w>1, %<w>2
1742   sub\\t%<w>0, %<w>1, #%n2
1743   #
1744   * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1745   #"
1746   "&& epilogue_completed
1747    && !reg_overlap_mentioned_p (operands[0], operands[1])
1748    && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1749   [(const_int 0)]
1750   {
1751     aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1752                               operands[2], operands[0], NULL_RTX);
1753     DONE;
1754   }
1755   ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1756   [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1757 )
1758
1759 (define_split
1760   [(set (match_operand:DI 0 "register_operand")
1761         (zero_extend:DI
1762           (plus:SI
1763             (match_operand:SI 1 "register_operand")
1764             (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1765   "epilogue_completed"
1766   [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1767    (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1768   {
1769     HOST_WIDE_INT i = INTVAL (operands[2]);
1770     HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1771     operands[3] = GEN_INT (i - s);
1772     operands[4] = GEN_INT (s);
1773     operands[5] = gen_lowpart (SImode, operands[0]);
1774   }
1775 )
1776
1777 (define_expand "addv<mode>4"
1778   [(match_operand:GPI 0 "register_operand")
1779    (match_operand:GPI 1 "register_operand")
1780    (match_operand:GPI 2 "register_operand")
1781    (label_ref (match_operand 3 "" ""))]
1782   ""
1783 {
1784   emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1785   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1786
1787   DONE;
1788 })
1789
1790 (define_expand "uaddv<mode>4"
1791   [(match_operand:GPI 0 "register_operand")
1792    (match_operand:GPI 1 "register_operand")
1793    (match_operand:GPI 2 "register_operand")
1794    (label_ref (match_operand 3 "" ""))]
1795   ""
1796 {
1797   emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1798   aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1799
1800   DONE;
1801 })
1802
1803 (define_expand "addti3"
1804   [(set (match_operand:TI 0 "register_operand" "")
1805         (plus:TI (match_operand:TI 1 "register_operand" "")
1806                  (match_operand:TI 2 "aarch64_reg_or_imm" "")))]
1807   ""
1808 {
1809   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1810
1811   aarch64_addti_scratch_regs (operands[1], operands[2],
1812                               &low_dest, &op1_low, &op2_low,
1813                               &high_dest, &op1_high, &op2_high);
1814
1815   if (op2_low == const0_rtx)
1816     {
1817       low_dest = op1_low;
1818       if (!aarch64_pluslong_operand (op2_high, DImode))
1819         op2_high = force_reg (DImode, op2_high);
1820       emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1821     }
1822   else
1823     {
1824       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1825                                       force_reg (DImode, op2_low)));
1826       emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1827                                      force_reg (DImode, op2_high)));
1828     }
1829
1830   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1831   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1832
1833   DONE;
1834 })
1835
1836 (define_expand "addvti4"
1837   [(match_operand:TI 0 "register_operand" "")
1838    (match_operand:TI 1 "register_operand" "")
1839    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1840    (label_ref (match_operand 3 "" ""))]
1841   ""
1842 {
1843   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1844
1845   aarch64_addti_scratch_regs (operands[1], operands[2],
1846                               &low_dest, &op1_low, &op2_low,
1847                               &high_dest, &op1_high, &op2_high);
1848
1849   if (op2_low == const0_rtx)
1850     {
1851       low_dest = op1_low;
1852       emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1853                                       force_reg (DImode, op2_high)));
1854     }
1855   else
1856     {
1857       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1858                                       force_reg (DImode, op2_low)));
1859       emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
1860                                       force_reg (DImode, op2_high)));
1861     }
1862
1863   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1864   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1865
1866   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1867   DONE;
1868 })
1869
1870 (define_expand "uaddvti4"
1871   [(match_operand:TI 0 "register_operand" "")
1872    (match_operand:TI 1 "register_operand" "")
1873    (match_operand:TI 2 "aarch64_reg_or_imm" "")
1874    (label_ref (match_operand 3 "" ""))]
1875   ""
1876 {
1877   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1878
1879   aarch64_addti_scratch_regs (operands[1], operands[2],
1880                               &low_dest, &op1_low, &op2_low,
1881                               &high_dest, &op1_high, &op2_high);
1882
1883   if (op2_low == const0_rtx)
1884     {
1885       low_dest = op1_low;
1886       emit_insn (gen_adddi3_compareC (high_dest, op1_high,
1887                                       force_reg (DImode, op2_high)));
1888     }
1889   else
1890     {
1891       emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1892                                       force_reg (DImode, op2_low)));
1893       emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
1894                                       force_reg (DImode, op2_high)));
1895     }
1896
1897   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1898   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1899
1900   aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1901   DONE;
1902  })
1903
1904 (define_insn "add<mode>3_compare0"
1905   [(set (reg:CC_NZ CC_REGNUM)
1906         (compare:CC_NZ
1907          (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1908                    (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1909          (const_int 0)))
1910    (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1911         (plus:GPI (match_dup 1) (match_dup 2)))]
1912   ""
1913   "@
1914   adds\\t%<w>0, %<w>1, %<w>2
1915   adds\\t%<w>0, %<w>1, %2
1916   subs\\t%<w>0, %<w>1, #%n2"
1917   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1918 )
1919
1920 ;; zero_extend version of above
1921 (define_insn "*addsi3_compare0_uxtw"
1922   [(set (reg:CC_NZ CC_REGNUM)
1923         (compare:CC_NZ
1924          (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1925                   (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1926          (const_int 0)))
1927    (set (match_operand:DI 0 "register_operand" "=r,r,r")
1928         (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1929   ""
1930   "@
1931   adds\\t%w0, %w1, %w2
1932   adds\\t%w0, %w1, %2
1933   subs\\t%w0, %w1, #%n2"
1934   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1935 )
1936
1937 (define_insn "*add<mode>3_compareC_cconly_imm"
1938   [(set (reg:CC_C CC_REGNUM)
1939         (ne:CC_C
1940           (plus:<DWI>
1941             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1942             (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1943           (zero_extend:<DWI>
1944             (plus:GPI
1945               (match_dup 0)
1946               (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1947   "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1948                                  <MODE>mode, operands[1])"
1949   "@
1950   cmn\\t%<w>0, %1
1951   cmp\\t%<w>0, #%n1"
1952   [(set_attr "type" "alus_imm")]
1953 )
1954
1955 (define_insn "*add<mode>3_compareC_cconly"
1956   [(set (reg:CC_C CC_REGNUM)
1957         (ne:CC_C
1958           (plus:<DWI>
1959             (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1960             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1961           (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1962   ""
1963   "cmn\\t%<w>0, %<w>1"
1964   [(set_attr "type" "alus_sreg")]
1965 )
1966
1967 (define_insn "*add<mode>3_compareC_imm"
1968   [(set (reg:CC_C CC_REGNUM)
1969         (ne:CC_C
1970           (plus:<DWI>
1971             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1972             (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1973           (zero_extend:<DWI>
1974             (plus:GPI
1975               (match_dup 1)
1976               (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1977    (set (match_operand:GPI 0 "register_operand" "=r,r")
1978         (plus:GPI (match_dup 1) (match_dup 2)))]
1979   "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1980                                  <MODE>mode, operands[2])"
1981   "@
1982   adds\\t%<w>0, %<w>1, %2
1983   subs\\t%<w>0, %<w>1, #%n2"
1984   [(set_attr "type" "alus_imm")]
1985 )
1986
1987 (define_insn "add<mode>3_compareC"
1988   [(set (reg:CC_C CC_REGNUM)
1989         (compare:CC_C
1990           (plus:<DWI>
1991             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1992             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1993           (zero_extend:<DWI>
1994             (plus:GPI (match_dup 1) (match_dup 2)))))
1995    (set (match_operand:GPI 0 "register_operand" "=r")
1996         (plus:GPI (match_dup 1) (match_dup 2)))]
1997   ""
1998   "adds\\t%<w>0, %<w>1, %<w>2"
1999   [(set_attr "type" "alus_sreg")]
2000 )
2001
2002 (define_insn "*add<mode>3_compareV_cconly_imm"
2003   [(set (reg:CC_V CC_REGNUM)
2004         (compare:CC_V
2005           (plus:<DWI>
2006             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
2007             (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
2008           (sign_extend:<DWI>
2009            (plus:GPI
2010             (match_dup 0)
2011             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2012   "INTVAL (operands[1]) == INTVAL (operands[2])"
2013   "@
2014   cmn\\t%<w>0, %<w>1
2015   cmp\\t%<w>0, #%n1"
2016   [(set_attr "type" "alus_imm")]
2017 )
2018
2019 (define_insn "*add<mode>3_compareV_cconly"
2020   [(set (reg:CC_V CC_REGNUM)
2021         (compare:CC_V
2022           (plus:<DWI>
2023             (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2024             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2025           (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2026   ""
2027   "cmn\\t%<w>0, %<w>1"
2028   [(set_attr "type" "alus_sreg")]
2029 )
2030
2031 (define_insn "*add<mode>3_compareV_imm"
2032   [(set (reg:CC_V CC_REGNUM)
2033         (compare:CC_V
2034           (plus:<DWI>
2035             (sign_extend:<DWI>
2036               (match_operand:GPI 1 "register_operand" "r,r"))
2037             (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2038           (sign_extend:<DWI>
2039             (plus:GPI (match_dup 1) (match_dup 2)))))
2040    (set (match_operand:GPI 0 "register_operand" "=r,r")
2041         (plus:GPI (match_dup 1) (match_dup 2)))]
2042    ""
2043    "@
2044    adds\\t%<w>0, %<w>1, %<w>2
2045    subs\\t%<w>0, %<w>1, #%n2"
2046   [(set_attr "type" "alus_imm,alus_imm")]
2047 )
2048
2049 (define_insn "add<mode>3_compareV"
2050   [(set (reg:CC_V CC_REGNUM)
2051         (compare:CC_V
2052           (plus:<DWI>
2053             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
2054             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2055           (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2056    (set (match_operand:GPI 0 "register_operand" "=r")
2057         (plus:GPI (match_dup 1) (match_dup 2)))]
2058   ""
2059   "adds\\t%<w>0, %<w>1, %<w>2"
2060   [(set_attr "type" "alus_sreg")]
2061 )
2062
2063 (define_insn "*adds_shift_imm_<mode>"
2064   [(set (reg:CC_NZ CC_REGNUM)
2065         (compare:CC_NZ
2066          (plus:GPI (ASHIFT:GPI
2067                     (match_operand:GPI 1 "register_operand" "r")
2068                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2069                    (match_operand:GPI 3 "register_operand" "r"))
2070          (const_int 0)))
2071    (set (match_operand:GPI 0 "register_operand" "=r")
2072         (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2073                   (match_dup 3)))]
2074   ""
2075   "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2076   [(set_attr "type" "alus_shift_imm")]
2077 )
2078
2079 (define_insn "*subs_shift_imm_<mode>"
2080   [(set (reg:CC_NZ CC_REGNUM)
2081         (compare:CC_NZ
2082          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2083                     (ASHIFT:GPI
2084                      (match_operand:GPI 2 "register_operand" "r")
2085                      (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2086          (const_int 0)))
2087    (set (match_operand:GPI 0 "register_operand" "=r")
2088         (minus:GPI (match_dup 1)
2089                    (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2090   ""
2091   "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2092   [(set_attr "type" "alus_shift_imm")]
2093 )
2094
2095 (define_insn "*adds_mul_imm_<mode>"
2096   [(set (reg:CC_NZ CC_REGNUM)
2097         (compare:CC_NZ
2098          (plus:GPI (mult:GPI
2099                     (match_operand:GPI 1 "register_operand" "r")
2100                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2101                    (match_operand:GPI 3 "register_operand" "r"))
2102          (const_int 0)))
2103    (set (match_operand:GPI 0 "register_operand" "=r")
2104         (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2105                   (match_dup 3)))]
2106   ""
2107   "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2108   [(set_attr "type" "alus_shift_imm")]
2109 )
2110
2111 (define_insn "*subs_mul_imm_<mode>"
2112   [(set (reg:CC_NZ CC_REGNUM)
2113         (compare:CC_NZ
2114          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2115                     (mult:GPI
2116                      (match_operand:GPI 2 "register_operand" "r")
2117                      (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2118          (const_int 0)))
2119    (set (match_operand:GPI 0 "register_operand" "=r")
2120         (minus:GPI (match_dup 1)
2121                    (mult:GPI (match_dup 2) (match_dup 3))))]
2122   ""
2123   "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2124   [(set_attr "type" "alus_shift_imm")]
2125 )
2126
2127 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2128   [(set (reg:CC_NZ CC_REGNUM)
2129         (compare:CC_NZ
2130          (plus:GPI
2131           (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2132           (match_operand:GPI 2 "register_operand" "r"))
2133         (const_int 0)))
2134    (set (match_operand:GPI 0 "register_operand" "=r")
2135         (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2136   ""
2137   "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2138   [(set_attr "type" "alus_ext")]
2139 )
2140
2141 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2142   [(set (reg:CC_NZ CC_REGNUM)
2143         (compare:CC_NZ
2144          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2145                     (ANY_EXTEND:GPI
2146                      (match_operand:ALLX 2 "register_operand" "r")))
2147         (const_int 0)))
2148    (set (match_operand:GPI 0 "register_operand" "=r")
2149         (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2150   ""
2151   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2152   [(set_attr "type" "alus_ext")]
2153 )
2154
2155 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2156   [(set (reg:CC_NZ CC_REGNUM)
2157         (compare:CC_NZ
2158          (plus:GPI (ashift:GPI
2159                     (ANY_EXTEND:GPI
2160                      (match_operand:ALLX 1 "register_operand" "r"))
2161                     (match_operand 2 "aarch64_imm3" "Ui3"))
2162                    (match_operand:GPI 3 "register_operand" "r"))
2163          (const_int 0)))
2164    (set (match_operand:GPI 0 "register_operand" "=rk")
2165         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2166                               (match_dup 2))
2167                   (match_dup 3)))]
2168   ""
2169   "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2170   [(set_attr "type" "alus_ext")]
2171 )
2172
2173 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2174   [(set (reg:CC_NZ CC_REGNUM)
2175         (compare:CC_NZ
2176          (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2177                     (ashift:GPI
2178                      (ANY_EXTEND:GPI
2179                       (match_operand:ALLX 2 "register_operand" "r"))
2180                      (match_operand 3 "aarch64_imm3" "Ui3")))
2181          (const_int 0)))
2182    (set (match_operand:GPI 0 "register_operand" "=rk")
2183         (minus:GPI (match_dup 1)
2184                    (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2185                                (match_dup 3))))]
2186   ""
2187   "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2188   [(set_attr "type" "alus_ext")]
2189 )
2190
2191 (define_insn "*adds_<optab><mode>_multp2"
2192   [(set (reg:CC_NZ CC_REGNUM)
2193         (compare:CC_NZ
2194          (plus:GPI (ANY_EXTRACT:GPI
2195                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2196                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2197                     (match_operand 3 "const_int_operand" "n")
2198                     (const_int 0))
2199                    (match_operand:GPI 4 "register_operand" "r"))
2200         (const_int 0)))
2201    (set (match_operand:GPI 0 "register_operand" "=r")
2202         (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2203                                    (match_dup 3)
2204                                    (const_int 0))
2205                   (match_dup 4)))]
2206   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2207   "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2208   [(set_attr "type" "alus_ext")]
2209 )
2210
2211 (define_insn "*subs_<optab><mode>_multp2"
2212   [(set (reg:CC_NZ CC_REGNUM)
2213         (compare:CC_NZ
2214          (minus:GPI (match_operand:GPI 4 "register_operand" "r")
2215                     (ANY_EXTRACT:GPI
2216                      (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2217                                (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2218                      (match_operand 3 "const_int_operand" "n")
2219                      (const_int 0)))
2220         (const_int 0)))
2221    (set (match_operand:GPI 0 "register_operand" "=r")
2222         (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2223                                   (mult:GPI (match_dup 1) (match_dup 2))
2224                                   (match_dup 3)
2225                                   (const_int 0))))]
2226   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2227   "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2228   [(set_attr "type" "alus_ext")]
2229 )
2230
2231 (define_insn "*add<mode>3nr_compare0"
2232   [(set (reg:CC_NZ CC_REGNUM)
2233         (compare:CC_NZ
2234          (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2235                    (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2236          (const_int 0)))]
2237   ""
2238   "@
2239   cmn\\t%<w>0, %<w>1
2240   cmn\\t%<w>0, %1
2241   cmp\\t%<w>0, #%n1"
2242   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2243 )
2244
2245 (define_insn "aarch64_sub<mode>_compare0"
2246   [(set (reg:CC_NZ CC_REGNUM)
2247         (compare:CC_NZ
2248          (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2249                    (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2250          (const_int 0)))]
2251   ""
2252   "cmp\\t%<w>0, %<w>1"
2253   [(set_attr "type" "alus_sreg")]
2254 )
2255
2256 (define_insn "*compare_neg<mode>"
2257   [(set (reg:CC_Z CC_REGNUM)
2258         (compare:CC_Z
2259          (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2260          (match_operand:GPI 1 "register_operand" "r")))]
2261   ""
2262   "cmn\\t%<w>1, %<w>0"
2263   [(set_attr "type" "alus_sreg")]
2264 )
2265
2266 (define_insn "*add_<shift>_<mode>"
2267   [(set (match_operand:GPI 0 "register_operand" "=r")
2268         (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2269                               (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2270                   (match_operand:GPI 3 "register_operand" "r")))]
2271   ""
2272   "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2273   [(set_attr "type" "alu_shift_imm")]
2274 )
2275
2276 ;; zero_extend version of above
2277 (define_insn "*add_<shift>_si_uxtw"
2278   [(set (match_operand:DI 0 "register_operand" "=r")
2279         (zero_extend:DI
2280          (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2281                              (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2282                   (match_operand:SI 3 "register_operand" "r"))))]
2283   ""
2284   "add\\t%w0, %w3, %w1, <shift> %2"
2285   [(set_attr "type" "alu_shift_imm")]
2286 )
2287
2288 (define_insn "*add_mul_imm_<mode>"
2289   [(set (match_operand:GPI 0 "register_operand" "=r")
2290         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2291                             (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2292                   (match_operand:GPI 3 "register_operand" "r")))]
2293   ""
2294   "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2295   [(set_attr "type" "alu_shift_imm")]
2296 )
2297
2298 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2299   [(set (match_operand:GPI 0 "register_operand" "=rk")
2300         (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2301                   (match_operand:GPI 2 "register_operand" "r")))]
2302   ""
2303   "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2304   [(set_attr "type" "alu_ext")]
2305 )
2306
2307 ;; zero_extend version of above
2308 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2309   [(set (match_operand:DI 0 "register_operand" "=rk")
2310         (zero_extend:DI
2311          (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2312                   (match_operand:GPI 2 "register_operand" "r"))))]
2313   ""
2314   "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2315   [(set_attr "type" "alu_ext")]
2316 )
2317
2318 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2319   [(set (match_operand:GPI 0 "register_operand" "=rk")
2320         (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2321                                (match_operand:ALLX 1 "register_operand" "r"))
2322                               (match_operand 2 "aarch64_imm3" "Ui3"))
2323                   (match_operand:GPI 3 "register_operand" "r")))]
2324   ""
2325   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2326   [(set_attr "type" "alu_ext")]
2327 )
2328
2329 ;; zero_extend version of above
2330 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2331   [(set (match_operand:DI 0 "register_operand" "=rk")
2332         (zero_extend:DI
2333          (plus:SI (ashift:SI (ANY_EXTEND:SI
2334                               (match_operand:SHORT 1 "register_operand" "r"))
2335                              (match_operand 2 "aarch64_imm3" "Ui3"))
2336                   (match_operand:SI 3 "register_operand" "r"))))]
2337   ""
2338   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2339   [(set_attr "type" "alu_ext")]
2340 )
2341
2342 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2343   [(set (match_operand:GPI 0 "register_operand" "=rk")
2344         (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2345                              (match_operand:ALLX 1 "register_operand" "r"))
2346                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2347                   (match_operand:GPI 3 "register_operand" "r")))]
2348   ""
2349   "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2350   [(set_attr "type" "alu_ext")]
2351 )
2352
2353 ;; zero_extend version of above
2354 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2355   [(set (match_operand:DI 0 "register_operand" "=rk")
2356         (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2357                              (match_operand:SHORT 1 "register_operand" "r"))
2358                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2359                   (match_operand:SI 3 "register_operand" "r"))))]
2360   ""
2361   "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2362   [(set_attr "type" "alu_ext")]
2363 )
2364
2365 (define_insn "*add_<optab><mode>_multp2"
2366   [(set (match_operand:GPI 0 "register_operand" "=rk")
2367         (plus:GPI (ANY_EXTRACT:GPI
2368                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2369                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2370                    (match_operand 3 "const_int_operand" "n")
2371                    (const_int 0))
2372                   (match_operand:GPI 4 "register_operand" "r")))]
2373   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2374   "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2375   [(set_attr "type" "alu_ext")]
2376 )
2377
2378 ;; zero_extend version of above
2379 (define_insn "*add_<optab>si_multp2_uxtw"
2380   [(set (match_operand:DI 0 "register_operand" "=rk")
2381         (zero_extend:DI
2382          (plus:SI (ANY_EXTRACT:SI
2383                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2384                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2385                    (match_operand 3 "const_int_operand" "n")
2386                    (const_int 0))
2387                   (match_operand:SI 4 "register_operand" "r"))))]
2388   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2389   "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2390   [(set_attr "type" "alu_ext")]
2391 )
2392
2393 (define_expand "add<mode>3_carryin"
2394   [(set (match_operand:GPI 0 "register_operand")
2395         (plus:GPI
2396           (plus:GPI
2397             (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2398             (match_operand:GPI 1 "aarch64_reg_or_zero"))
2399           (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2400    ""
2401    ""
2402 )
2403
2404 ;; Note that add with carry with two zero inputs is matched by cset,
2405 ;; and that add with carry with one zero input is matched by cinc.
2406
2407 (define_insn "*add<mode>3_carryin"
2408   [(set (match_operand:GPI 0 "register_operand" "=r")
2409         (plus:GPI
2410           (plus:GPI
2411             (match_operand:GPI 3 "aarch64_carry_operation" "")
2412             (match_operand:GPI 1 "register_operand" "r"))
2413           (match_operand:GPI 2 "register_operand" "r")))]
2414    ""
2415    "adc\\t%<w>0, %<w>1, %<w>2"
2416   [(set_attr "type" "adc_reg")]
2417 )
2418
2419 ;; zero_extend version of above
2420 (define_insn "*addsi3_carryin_uxtw"
2421   [(set (match_operand:DI 0 "register_operand" "=r")
2422         (zero_extend:DI
2423           (plus:SI
2424             (plus:SI
2425               (match_operand:SI 3 "aarch64_carry_operation" "")
2426               (match_operand:SI 1 "register_operand" "r"))
2427             (match_operand:SI 2 "register_operand" "r"))))]
2428    ""
2429    "adc\\t%w0, %w1, %w2"
2430   [(set_attr "type" "adc_reg")]
2431 )
2432
2433 (define_expand "add<mode>3_carryinC"
2434   [(parallel
2435      [(set (match_dup 3)
2436            (compare:CC_C
2437              (plus:<DWI>
2438                (plus:<DWI>
2439                  (match_dup 4)
2440                  (zero_extend:<DWI>
2441                    (match_operand:GPI 1 "register_operand" "")))
2442                (zero_extend:<DWI>
2443                  (match_operand:GPI 2 "register_operand" "")))
2444            (zero_extend:<DWI>
2445              (plus:GPI
2446                (plus:GPI (match_dup 5) (match_dup 1))
2447                (match_dup 2)))))
2448       (set (match_operand:GPI 0 "register_operand")
2449            (plus:GPI
2450              (plus:GPI (match_dup 5) (match_dup 1))
2451              (match_dup 2)))])]
2452    ""
2453 {
2454   operands[3] = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2455   operands[4] = gen_rtx_NE (<DWI>mode, operands[3], const0_rtx);
2456   operands[5] = gen_rtx_NE (<MODE>mode, operands[3], const0_rtx);
2457 })
2458
2459 (define_insn "*add<mode>3_carryinC_zero"
2460   [(set (reg:CC_C CC_REGNUM)
2461         (compare:CC_C
2462           (plus:<DWI>
2463             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2464             (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2465           (zero_extend:<DWI>
2466             (plus:GPI
2467               (match_operand:GPI 3 "aarch64_carry_operation" "")
2468               (match_dup 1)))))
2469    (set (match_operand:GPI 0 "register_operand" "=r")
2470         (plus:GPI (match_dup 3) (match_dup 1)))]
2471    ""
2472    "adcs\\t%<w>0, %<w>1, <w>zr"
2473   [(set_attr "type" "adc_reg")]
2474 )
2475
2476 (define_insn "*add<mode>3_carryinC"
2477   [(set (reg:CC_C CC_REGNUM)
2478         (compare:CC_C
2479           (plus:<DWI>
2480             (plus:<DWI>
2481               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2482               (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2483             (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2484           (zero_extend:<DWI>
2485             (plus:GPI
2486               (plus:GPI
2487                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2488                 (match_dup 1))
2489               (match_dup 2)))))
2490    (set (match_operand:GPI 0 "register_operand" "=r")
2491         (plus:GPI
2492           (plus:GPI (match_dup 4) (match_dup 1))
2493           (match_dup 2)))]
2494    ""
2495    "adcs\\t%<w>0, %<w>1, %<w>2"
2496   [(set_attr "type" "adc_reg")]
2497 )
2498
2499 (define_expand "add<mode>3_carryinV"
2500   [(parallel
2501      [(set (reg:CC_V CC_REGNUM)
2502            (compare:CC_V
2503              (plus:<DWI>
2504                (plus:<DWI>
2505                  (match_dup 3)
2506                  (sign_extend:<DWI>
2507                    (match_operand:GPI 1 "register_operand" "")))
2508                (sign_extend:<DWI>
2509                  (match_operand:GPI 2 "register_operand" "")))
2510            (sign_extend:<DWI>
2511              (plus:GPI
2512                (plus:GPI (match_dup 4) (match_dup 1))
2513                (match_dup 2)))))
2514       (set (match_operand:GPI 0 "register_operand")
2515            (plus:GPI
2516              (plus:GPI (match_dup 4) (match_dup 1))
2517              (match_dup 2)))])]
2518    ""
2519 {
2520   rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2521   operands[3] = gen_rtx_NE (<DWI>mode, cc, const0_rtx);
2522   operands[4] = gen_rtx_NE (<MODE>mode, cc, const0_rtx);
2523 })
2524
2525 (define_insn "*add<mode>3_carryinV_zero"
2526   [(set (reg:CC_V CC_REGNUM)
2527         (compare:CC_V
2528           (plus:<DWI>
2529             (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2530             (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2531           (sign_extend:<DWI>
2532             (plus:GPI
2533               (match_operand:GPI 3 "aarch64_carry_operation" "")
2534               (match_dup 1)))))
2535    (set (match_operand:GPI 0 "register_operand" "=r")
2536         (plus:GPI (match_dup 3) (match_dup 1)))]
2537    ""
2538    "adcs\\t%<w>0, %<w>1, <w>zr"
2539   [(set_attr "type" "adc_reg")]
2540 )
2541
2542 (define_insn "*add<mode>3_carryinV"
2543   [(set (reg:CC_V CC_REGNUM)
2544         (compare:CC_V
2545           (plus:<DWI>
2546             (plus:<DWI>
2547               (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2548               (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2549             (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2550           (sign_extend:<DWI>
2551             (plus:GPI
2552               (plus:GPI
2553                 (match_operand:GPI 4 "aarch64_carry_operation" "")
2554                 (match_dup 1))
2555               (match_dup 2)))))
2556    (set (match_operand:GPI 0 "register_operand" "=r")
2557         (plus:GPI
2558           (plus:GPI (match_dup 4) (match_dup 1))
2559           (match_dup 2)))]
2560    ""
2561    "adcs\\t%<w>0, %<w>1, %<w>2"
2562   [(set_attr "type" "adc_reg")]
2563 )
2564
2565 (define_insn "*add_uxt<mode>_shift2"
2566   [(set (match_operand:GPI 0 "register_operand" "=rk")
2567         (plus:GPI (and:GPI
2568                    (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2569                                (match_operand 2 "aarch64_imm3" "Ui3"))
2570                    (match_operand 3 "const_int_operand" "n"))
2571                   (match_operand:GPI 4 "register_operand" "r")))]
2572   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2573   "*
2574   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2575                                            INTVAL (operands[3])));
2576   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2577   [(set_attr "type" "alu_ext")]
2578 )
2579
2580 ;; zero_extend version of above
2581 (define_insn "*add_uxtsi_shift2_uxtw"
2582   [(set (match_operand:DI 0 "register_operand" "=rk")
2583         (zero_extend:DI
2584          (plus:SI (and:SI
2585                    (ashift:SI (match_operand:SI 1 "register_operand" "r")
2586                               (match_operand 2 "aarch64_imm3" "Ui3"))
2587                    (match_operand 3 "const_int_operand" "n"))
2588                   (match_operand:SI 4 "register_operand" "r"))))]
2589   "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2590   "*
2591   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2592                                            INTVAL (operands[3])));
2593   return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2594   [(set_attr "type" "alu_ext")]
2595 )
2596
2597 (define_insn "*add_uxt<mode>_multp2"
2598   [(set (match_operand:GPI 0 "register_operand" "=rk")
2599         (plus:GPI (and:GPI
2600                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2601                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2602                    (match_operand 3 "const_int_operand" "n"))
2603                   (match_operand:GPI 4 "register_operand" "r")))]
2604   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2605   "*
2606   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2607                                            INTVAL (operands[3])));
2608   return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2609   [(set_attr "type" "alu_ext")]
2610 )
2611
2612 ;; zero_extend version of above
2613 (define_insn "*add_uxtsi_multp2_uxtw"
2614   [(set (match_operand:DI 0 "register_operand" "=rk")
2615         (zero_extend:DI
2616          (plus:SI (and:SI
2617                    (mult:SI (match_operand:SI 1 "register_operand" "r")
2618                             (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2619                    (match_operand 3 "const_int_operand" "n"))
2620                   (match_operand:SI 4 "register_operand" "r"))))]
2621   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2622   "*
2623   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2624                                            INTVAL (operands[3])));
2625   return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2626   [(set_attr "type" "alu_ext")]
2627 )
2628
2629 (define_insn "subsi3"
2630   [(set (match_operand:SI 0 "register_operand" "=rk")
2631         (minus:SI (match_operand:SI 1 "register_operand" "rk")
2632                   (match_operand:SI 2 "register_operand" "r")))]
2633   ""
2634   "sub\\t%w0, %w1, %w2"
2635   [(set_attr "type" "alu_sreg")]
2636 )
2637
2638 ;; zero_extend version of above
2639 (define_insn "*subsi3_uxtw"
2640   [(set (match_operand:DI 0 "register_operand" "=rk")
2641         (zero_extend:DI
2642          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2643                    (match_operand:SI 2 "register_operand" "r"))))]
2644   ""
2645   "sub\\t%w0, %w1, %w2"
2646   [(set_attr "type" "alu_sreg")]
2647 )
2648
2649 (define_insn "subdi3"
2650   [(set (match_operand:DI 0 "register_operand" "=rk,w")
2651         (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2652                   (match_operand:DI 2 "register_operand" "r,w")))]
2653   ""
2654   "@
2655    sub\\t%x0, %x1, %x2
2656    sub\\t%d0, %d1, %d2"
2657   [(set_attr "type" "alu_sreg, neon_sub")
2658    (set_attr "arch" "*,simd")]
2659 )
2660
2661 (define_expand "subv<mode>4"
2662   [(match_operand:GPI 0 "register_operand")
2663    (match_operand:GPI 1 "aarch64_reg_or_zero")
2664    (match_operand:GPI 2 "aarch64_reg_or_zero")
2665    (label_ref (match_operand 3 "" ""))]
2666   ""
2667 {
2668   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2669   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2670
2671   DONE;
2672 })
2673
2674 (define_expand "usubv<mode>4"
2675   [(match_operand:GPI 0 "register_operand")
2676    (match_operand:GPI 1 "aarch64_reg_or_zero")
2677    (match_operand:GPI 2 "aarch64_reg_or_zero")
2678    (label_ref (match_operand 3 "" ""))]
2679   ""
2680 {
2681   emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2682   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2683
2684   DONE;
2685 })
2686
2687 (define_expand "subti3"
2688   [(set (match_operand:TI 0 "register_operand" "")
2689         (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero" "")
2690                   (match_operand:TI 2 "register_operand" "")))]
2691   ""
2692 {
2693   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2694
2695   aarch64_subvti_scratch_regs (operands[1], operands[2],
2696                                &low_dest, &op1_low, &op2_low,
2697                                &high_dest, &op1_high, &op2_high);
2698
2699   emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2700   emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2701
2702   emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2703   emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2704   DONE;
2705 })
2706
2707 (define_expand "subvti4"
2708   [(match_operand:TI 0 "register_operand")
2709    (match_operand:TI 1 "aarch64_reg_or_zero")
2710    (match_operand:TI 2 "aarch64_reg_or_imm")
2711    (label_ref (match_operand 3 "" ""))]
2712   ""
2713 {
2714   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2715
2716   aarch64_subvti_scratch_regs (operands[1], operands[2],
2717                                &low_dest, &op1_low, &op2_low,
2718                                &high_dest, &op1_high, &op2_high);
2719   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2720                          high_dest, op1_high, op2_high);
2721
2722   aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2723   DONE;
2724 })
2725
2726 (define_expand "usubvti4"
2727   [(match_operand:TI 0 "register_operand")
2728    (match_operand:TI 1 "aarch64_reg_or_zero")
2729    (match_operand:TI 2 "aarch64_reg_or_imm")
2730    (label_ref (match_operand 3 "" ""))]
2731   ""
2732 {
2733   rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2734
2735   aarch64_subvti_scratch_regs (operands[1], operands[2],
2736                                     &low_dest, &op1_low, &op2_low,
2737                                &high_dest, &op1_high, &op2_high);
2738   aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2739                          high_dest, op1_high, op2_high);
2740
2741   aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2742   DONE;
2743 })
2744
2745 (define_insn "*sub<mode>3_compare0"
2746   [(set (reg:CC_NZ CC_REGNUM)
2747         (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2748                                   (match_operand:GPI 2 "register_operand" "r"))
2749                        (const_int 0)))
2750    (set (match_operand:GPI 0 "register_operand" "=r")
2751         (minus:GPI (match_dup 1) (match_dup 2)))]
2752   ""
2753   "subs\\t%<w>0, %<w>1, %<w>2"
2754   [(set_attr "type" "alus_sreg")]
2755 )
2756
2757 ;; zero_extend version of above
2758 (define_insn "*subsi3_compare0_uxtw"
2759   [(set (reg:CC_NZ CC_REGNUM)
2760         (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2761                                  (match_operand:SI 2 "register_operand" "r"))
2762                        (const_int 0)))
2763    (set (match_operand:DI 0 "register_operand" "=r")
2764         (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2765   ""
2766   "subs\\t%w0, %w1, %w2"
2767   [(set_attr "type" "alus_sreg")]
2768 )
2769
2770 (define_insn "*sub<mode>3_compare1_imm"
2771   [(set (reg:CC CC_REGNUM)
2772         (compare:CC
2773           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ,rZ")
2774           (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2775    (set (match_operand:GPI 0 "register_operand" "=r,r")
2776         (plus:GPI
2777           (match_dup 1)
2778           (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
2779   "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
2780   "@
2781   subs\\t%<w>0, %<w>1, #%n3
2782   adds\\t%<w>0, %<w>1, %3"
2783   [(set_attr "type" "alus_imm")]
2784 )
2785
2786 (define_insn "sub<mode>3_compare1"
2787   [(set (reg:CC CC_REGNUM)
2788         (compare:CC
2789           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2790           (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2791    (set (match_operand:GPI 0 "register_operand" "=r")
2792         (minus:GPI (match_dup 1) (match_dup 2)))]
2793   ""
2794   "subs\\t%<w>0, %<w>1, %<w>2"
2795   [(set_attr "type" "alus_sreg")]
2796 )
2797
2798 (define_insn "sub<mode>3_compare1_imm"
2799   [(set (reg:CC CC_REGNUM)
2800         (compare:CC
2801           (match_operand:GPI 1 "register_operand" "r")
2802           (match_operand:GPI 3 "const_int_operand" "n")))
2803    (set (match_operand:GPI 0 "register_operand" "=r")
2804         (plus:GPI (match_dup 1)
2805                   (match_operand:GPI 2 "aarch64_sub_immediate" "J")))]
2806   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2807   "subs\\t%<w>0, %<w>1, #%n2"
2808   [(set_attr "type" "alus_sreg")]
2809 )
2810
2811 (define_peephole2
2812   [(set (match_operand:GPI 0 "register_operand")
2813         (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
2814                     (match_operand:GPI 2 "aarch64_reg_or_zero")))
2815    (set (reg:CC CC_REGNUM)
2816         (compare:CC
2817           (match_dup 1)
2818           (match_dup 2)))]
2819   "!reg_overlap_mentioned_p (operands[0], operands[1])
2820    && !reg_overlap_mentioned_p (operands[0], operands[2])"
2821   [(const_int 0)]
2822   {
2823     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2824                                          operands[2]));
2825     DONE;
2826   }
2827 )
2828
2829 ;; Same as the above peephole but with the compare and minus in
2830 ;; swapped order.  The restriction on overlap between operand 0
2831 ;; and operands 1 and 2 doesn't apply here.
2832 (define_peephole2
2833   [(set (reg:CC CC_REGNUM)
2834         (compare:CC
2835           (match_operand:GPI 1 "aarch64_reg_or_zero")
2836           (match_operand:GPI 2 "aarch64_reg_or_zero")))
2837    (set (match_operand:GPI 0 "register_operand")
2838         (minus:GPI (match_dup 1)
2839                    (match_dup 2)))]
2840   ""
2841   [(const_int 0)]
2842   {
2843     emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2844                                          operands[2]));
2845     DONE;
2846   }
2847 )
2848
2849 (define_peephole2
2850   [(set (match_operand:GPI 0 "register_operand")
2851         (plus:GPI (match_operand:GPI 1 "register_operand")
2852                   (match_operand:GPI 2 "aarch64_sub_immediate")))
2853    (set (reg:CC CC_REGNUM)
2854         (compare:CC
2855           (match_dup 1)
2856           (match_operand:GPI 3 "const_int_operand")))]
2857   "!reg_overlap_mentioned_p (operands[0], operands[1])
2858    && INTVAL (operands[3]) == -INTVAL (operands[2])"
2859   [(const_int 0)]
2860   {
2861     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2862                                          operands[2], operands[3]));
2863     DONE;
2864   }
2865 )
2866
2867 ;; Same as the above peephole but with the compare and minus in
2868 ;; swapped order.  The restriction on overlap between operand 0
2869 ;; and operands 1 doesn't apply here.
2870 (define_peephole2
2871   [(set (reg:CC CC_REGNUM)
2872         (compare:CC
2873           (match_operand:GPI 1 "register_operand")
2874           (match_operand:GPI 3 "const_int_operand")))
2875    (set (match_operand:GPI 0 "register_operand")
2876         (plus:GPI (match_dup 1)
2877                   (match_operand:GPI 2 "aarch64_sub_immediate")))]
2878   "INTVAL (operands[3]) == -INTVAL (operands[2])"
2879   [(const_int 0)]
2880   {
2881     emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2882                                          operands[2], operands[3]));
2883     DONE;
2884   }
2885 )
2886
2887 (define_insn "*sub_<shift>_<mode>"
2888   [(set (match_operand:GPI 0 "register_operand" "=r")
2889         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2890                    (ASHIFT:GPI
2891                     (match_operand:GPI 1 "register_operand" "r")
2892                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2893   ""
2894   "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2895   [(set_attr "type" "alu_shift_imm")]
2896 )
2897
2898 ;; zero_extend version of above
2899 (define_insn "*sub_<shift>_si_uxtw"
2900   [(set (match_operand:DI 0 "register_operand" "=r")
2901         (zero_extend:DI
2902          (minus:SI (match_operand:SI 3 "register_operand" "r")
2903                    (ASHIFT:SI
2904                     (match_operand:SI 1 "register_operand" "r")
2905                     (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2906   ""
2907   "sub\\t%w0, %w3, %w1, <shift> %2"
2908   [(set_attr "type" "alu_shift_imm")]
2909 )
2910
2911 (define_insn "*sub_mul_imm_<mode>"
2912   [(set (match_operand:GPI 0 "register_operand" "=r")
2913         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2914                    (mult:GPI
2915                     (match_operand:GPI 1 "register_operand" "r")
2916                     (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2917   ""
2918   "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2919   [(set_attr "type" "alu_shift_imm")]
2920 )
2921
2922 ;; zero_extend version of above
2923 (define_insn "*sub_mul_imm_si_uxtw"
2924   [(set (match_operand:DI 0 "register_operand" "=r")
2925         (zero_extend:DI
2926          (minus:SI (match_operand:SI 3 "register_operand" "r")
2927                    (mult:SI
2928                     (match_operand:SI 1 "register_operand" "r")
2929                     (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2930   ""
2931   "sub\\t%w0, %w3, %w1, lsl %p2"
2932   [(set_attr "type" "alu_shift_imm")]
2933 )
2934
2935 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2936   [(set (match_operand:GPI 0 "register_operand" "=rk")
2937         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2938                    (ANY_EXTEND:GPI
2939                     (match_operand:ALLX 2 "register_operand" "r"))))]
2940   ""
2941   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2942   [(set_attr "type" "alu_ext")]
2943 )
2944
2945 ;; zero_extend version of above
2946 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2947   [(set (match_operand:DI 0 "register_operand" "=rk")
2948         (zero_extend:DI
2949          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2950                    (ANY_EXTEND:SI
2951                     (match_operand:SHORT 2 "register_operand" "r")))))]
2952   ""
2953   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2954   [(set_attr "type" "alu_ext")]
2955 )
2956
2957 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2958   [(set (match_operand:GPI 0 "register_operand" "=rk")
2959         (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2960                    (ashift:GPI (ANY_EXTEND:GPI
2961                                 (match_operand:ALLX 2 "register_operand" "r"))
2962                                (match_operand 3 "aarch64_imm3" "Ui3"))))]
2963   ""
2964   "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2965   [(set_attr "type" "alu_ext")]
2966 )
2967
2968 ;; zero_extend version of above
2969 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2970   [(set (match_operand:DI 0 "register_operand" "=rk")
2971         (zero_extend:DI
2972          (minus:SI (match_operand:SI 1 "register_operand" "rk")
2973                    (ashift:SI (ANY_EXTEND:SI
2974                                (match_operand:SHORT 2 "register_operand" "r"))
2975                               (match_operand 3 "aarch64_imm3" "Ui3")))))]
2976   ""
2977   "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2978   [(set_attr "type" "alu_ext")]
2979 )
2980
2981 (define_insn "*sub_<optab><mode>_multp2"
2982   [(set (match_operand:GPI 0 "register_operand" "=rk")
2983         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2984                    (ANY_EXTRACT:GPI
2985                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2986                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2987                     (match_operand 3 "const_int_operand" "n")
2988                     (const_int 0))))]
2989   "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2990   "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2991   [(set_attr "type" "alu_ext")]
2992 )
2993
2994 ;; zero_extend version of above
2995 (define_insn "*sub_<optab>si_multp2_uxtw"
2996   [(set (match_operand:DI 0 "register_operand" "=rk")
2997         (zero_extend:DI
2998          (minus:SI (match_operand:SI 4 "register_operand" "rk")
2999                    (ANY_EXTRACT:SI
3000                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3001                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3002                     (match_operand 3 "const_int_operand" "n")
3003                     (const_int 0)))))]
3004   "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
3005   "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
3006   [(set_attr "type" "alu_ext")]
3007 )
3008
3009 ;; The hardware description is op1 + ~op2 + C.
3010 ;;                           = op1 + (-op2 + 1) + (1 - !C)
3011 ;;                           = op1 - op2 - 1 + 1 - !C
3012 ;;                           = op1 - op2 - !C.
3013 ;; We describe the latter.
3014
3015 (define_insn "*sub<mode>3_carryin0"
3016   [(set (match_operand:GPI 0 "register_operand" "=r")
3017         (minus:GPI
3018           (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3019           (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3020    ""
3021    "sbc\\t%<w>0, %<w>1, <w>zr"
3022   [(set_attr "type" "adc_reg")]
3023 )
3024
3025 ;; zero_extend version of the above
3026 (define_insn "*subsi3_carryin_uxtw"
3027   [(set (match_operand:DI 0 "register_operand" "=r")
3028         (zero_extend:DI
3029           (minus:SI
3030             (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3031             (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3032    ""
3033    "sbc\\t%w0, %w1, wzr"
3034   [(set_attr "type" "adc_reg")]
3035 )
3036
3037 (define_expand "sub<mode>3_carryin"
3038   [(set (match_operand:GPI 0 "register_operand")
3039         (minus:GPI
3040           (minus:GPI
3041             (match_operand:GPI 1 "aarch64_reg_or_zero")
3042             (match_operand:GPI 2 "register_operand"))
3043           (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3044    ""
3045    ""
3046 )
3047
3048 (define_insn "*sub<mode>3_carryin"
3049   [(set (match_operand:GPI 0 "register_operand" "=r")
3050         (minus:GPI
3051           (minus:GPI
3052             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3053             (match_operand:GPI 2 "register_operand" "r"))
3054           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3055
3056    ""
3057    "sbc\\t%<w>0, %<w>1, %<w>2"
3058   [(set_attr "type" "adc_reg")]
3059 )
3060
3061 ;; zero_extend version of the above
3062 (define_insn "*subsi3_carryin_uxtw"
3063   [(set (match_operand:DI 0 "register_operand" "=r")
3064         (zero_extend:DI
3065           (minus:SI
3066             (minus:SI
3067               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3068               (match_operand:SI 2 "register_operand" "r"))
3069             (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3070
3071    ""
3072    "sbc\\t%w0, %w1, %w2"
3073   [(set_attr "type" "adc_reg")]
3074 )
3075
3076 (define_insn "*sub<mode>3_carryin_alt"
3077   [(set (match_operand:GPI 0 "register_operand" "=r")
3078         (minus:GPI
3079           (minus:GPI
3080             (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3081             (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3082           (match_operand:GPI 2 "register_operand" "r")))]
3083    ""
3084    "sbc\\t%<w>0, %<w>1, %<w>2"
3085   [(set_attr "type" "adc_reg")]
3086 )
3087
3088 ;; zero_extend version of the above
3089 (define_insn "*subsi3_carryin_alt_uxtw"
3090   [(set (match_operand:DI 0 "register_operand" "=r")
3091         (zero_extend:DI
3092           (minus:SI
3093             (minus:SI
3094               (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3095               (match_operand:SI 3 "aarch64_borrow_operation" ""))
3096             (match_operand:SI 2 "register_operand" "r"))))]
3097    ""
3098    "sbc\\t%w0, %w1, %w2"
3099   [(set_attr "type" "adc_reg")]
3100 )
3101
3102 (define_expand "sub<mode>3_carryinCV"
3103   [(parallel
3104      [(set (reg:CC CC_REGNUM)
3105            (compare:CC
3106              (sign_extend:<DWI>
3107                (match_operand:GPI 1 "aarch64_reg_or_zero" ""))
3108              (plus:<DWI>
3109                (sign_extend:<DWI>
3110                  (match_operand:GPI 2 "register_operand" ""))
3111                (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3112       (set (match_operand:GPI 0 "register_operand" "")
3113            (minus:GPI
3114              (minus:GPI (match_dup 1) (match_dup 2))
3115              (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3116    ""
3117 )
3118
3119 (define_insn "*sub<mode>3_carryinCV_z1_z2"
3120   [(set (reg:CC CC_REGNUM)
3121         (compare:CC
3122           (const_int 0)
3123           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3124    (set (match_operand:GPI 0 "register_operand" "=r")
3125         (neg:GPI (match_operand:GPI 1 "aarch64_borrow_operation" "")))]
3126    ""
3127    "sbcs\\t%<w>0, <w>zr, <w>zr"
3128   [(set_attr "type" "adc_reg")]
3129 )
3130
3131 (define_insn "*sub<mode>3_carryinCV_z1"
3132   [(set (reg:CC CC_REGNUM)
3133         (compare:CC
3134           (const_int 0)
3135           (plus:<DWI>
3136             (sign_extend:<DWI>
3137               (match_operand:GPI 1 "register_operand" "r"))
3138             (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3139    (set (match_operand:GPI 0 "register_operand" "=r")
3140         (minus:GPI
3141           (neg:GPI (match_dup 1))
3142           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3143    ""
3144    "sbcs\\t%<w>0, <w>zr, %<w>1"
3145   [(set_attr "type" "adc_reg")]
3146 )
3147
3148 (define_insn "*sub<mode>3_carryinCV_z2"
3149   [(set (reg:CC CC_REGNUM)
3150         (compare:CC
3151           (sign_extend:<DWI>
3152             (match_operand:GPI 1 "register_operand" "r"))
3153           (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3154    (set (match_operand:GPI 0 "register_operand" "=r")
3155         (minus:GPI
3156           (match_dup 1)
3157           (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3158    ""
3159    "sbcs\\t%<w>0, %<w>1, <w>zr"
3160   [(set_attr "type" "adc_reg")]
3161 )
3162
3163 (define_insn "*sub<mode>3_carryinCV"
3164   [(set (reg:CC CC_REGNUM)
3165         (compare:CC
3166           (sign_extend:<DWI>
3167             (match_operand:GPI 1 "register_operand" "r"))
3168           (plus:<DWI>
3169             (sign_extend:<DWI>
3170               (match_operand:GPI 2 "register_operand" "r"))
3171             (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3172    (set (match_operand:GPI 0 "register_operand" "=r")
3173         (minus:GPI
3174           (minus:GPI (match_dup 1) (match_dup 2))
3175           (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3176    ""
3177    "sbcs\\t%<w>0, %<w>1, %<w>2"
3178   [(set_attr "type" "adc_reg")]
3179 )
3180
3181 (define_insn "*sub_uxt<mode>_shift2"
3182   [(set (match_operand:GPI 0 "register_operand" "=rk")
3183         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3184                    (and:GPI
3185                     (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3186                                 (match_operand 2 "aarch64_imm3" "Ui3"))
3187                     (match_operand 3 "const_int_operand" "n"))))]
3188   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3189   "*
3190   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3191                                            INTVAL (operands[3])));
3192   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3193   [(set_attr "type" "alu_ext")]
3194 )
3195
3196 ;; zero_extend version of above
3197 (define_insn "*sub_uxtsi_shift2_uxtw"
3198   [(set (match_operand:DI 0 "register_operand" "=rk")
3199         (zero_extend:DI
3200          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3201                    (and:SI
3202                     (ashift:SI (match_operand:SI 1 "register_operand" "r")
3203                                (match_operand 2 "aarch64_imm3" "Ui3"))
3204                     (match_operand 3 "const_int_operand" "n")))))]
3205   "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3206   "*
3207   operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3208                                            INTVAL (operands[3])));
3209   return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3210   [(set_attr "type" "alu_ext")]
3211 )
3212
3213 (define_insn "*sub_uxt<mode>_multp2"
3214   [(set (match_operand:GPI 0 "register_operand" "=rk")
3215         (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3216                    (and:GPI
3217                     (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3218                               (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3219                     (match_operand 3 "const_int_operand" "n"))))]
3220   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3221   "*
3222   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3223                                            INTVAL (operands[3])));
3224   return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3225   [(set_attr "type" "alu_ext")]
3226 )
3227
3228 ;; zero_extend version of above
3229 (define_insn "*sub_uxtsi_multp2_uxtw"
3230   [(set (match_operand:DI 0 "register_operand" "=rk")
3231         (zero_extend:DI
3232          (minus:SI (match_operand:SI 4 "register_operand" "rk")
3233                    (and:SI
3234                     (mult:SI (match_operand:SI 1 "register_operand" "r")
3235                              (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3236                     (match_operand 3 "const_int_operand" "n")))))]
3237   "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3238   "*
3239   operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3240                                            INTVAL (operands[3])));
3241   return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3242   [(set_attr "type" "alu_ext")]
3243 )
3244
3245 (define_expand "abs<mode>2"
3246   [(match_operand:GPI 0 "register_operand" "")
3247    (match_operand:GPI 1 "register_operand" "")]
3248   ""
3249   {
3250     rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3251     rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3252     emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3253     DONE;
3254   }
3255 )
3256
3257 (define_insn "neg<mode>2"
3258   [(set (match_operand:GPI 0 "register_operand" "=r,w")
3259         (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3260   ""
3261   "@
3262    neg\\t%<w>0, %<w>1
3263    neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3264   [(set_attr "type" "alu_sreg, neon_neg<q>")
3265    (set_attr "arch" "*,simd")]
3266 )
3267
3268 ;; zero_extend version of above
3269 (define_insn "*negsi2_uxtw"
3270   [(set (match_operand:DI 0 "register_operand" "=r")
3271         (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3272   ""
3273   "neg\\t%w0, %w1"
3274   [(set_attr "type" "alu_sreg")]
3275 )
3276
3277 (define_insn "*ngc<mode>"
3278   [(set (match_operand:GPI 0 "register_operand" "=r")
3279         (minus:GPI
3280           (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3281           (match_operand:GPI 1 "register_operand" "r")))]
3282   ""
3283   "ngc\\t%<w>0, %<w>1"
3284   [(set_attr "type" "adc_reg")]
3285 )
3286
3287 (define_insn "*ngcsi_uxtw"
3288   [(set (match_operand:DI 0 "register_operand" "=r")
3289         (zero_extend:DI
3290           (minus:SI
3291             (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3292             (match_operand:SI 1 "register_operand" "r"))))]
3293   ""
3294   "ngc\\t%w0, %w1"
3295   [(set_attr "type" "adc_reg")]
3296 )
3297
3298 (define_insn "neg<mode>2_compare0"
3299   [(set (reg:CC_NZ CC_REGNUM)
3300         (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3301                        (const_int 0)))
3302    (set (match_operand:GPI 0 "register_operand" "=r")
3303         (neg:GPI (match_dup 1)))]
3304   ""
3305   "negs\\t%<w>0, %<w>1"
3306   [(set_attr "type" "alus_sreg")]
3307 )
3308
3309 ;; zero_extend version of above
3310 (define_insn "*negsi2_compare0_uxtw"
3311   [(set (reg:CC_NZ CC_REGNUM)
3312         (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3313                        (const_int 0)))
3314    (set (match_operand:DI 0 "register_operand" "=r")
3315         (zero_extend:DI (neg:SI (match_dup 1))))]
3316   ""
3317   "negs\\t%w0, %w1"
3318   [(set_attr "type" "alus_sreg")]
3319 )
3320
3321 (define_insn "*neg_<shift><mode>3_compare0"
3322   [(set (reg:CC_NZ CC_REGNUM)
3323         (compare:CC_NZ
3324          (neg:GPI (ASHIFT:GPI
3325                    (match_operand:GPI 1 "register_operand" "r")
3326                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3327          (const_int 0)))
3328    (set (match_operand:GPI 0 "register_operand" "=r")
3329         (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3330   ""
3331   "negs\\t%<w>0, %<w>1, <shift> %2"
3332   [(set_attr "type" "alus_shift_imm")]
3333 )
3334
3335 (define_insn "*neg_<shift>_<mode>2"
3336   [(set (match_operand:GPI 0 "register_operand" "=r")
3337         (neg:GPI (ASHIFT:GPI
3338                   (match_operand:GPI 1 "register_operand" "r")
3339                   (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3340   ""
3341   "neg\\t%<w>0, %<w>1, <shift> %2"
3342   [(set_attr "type" "alu_shift_imm")]
3343 )
3344
3345 ;; zero_extend version of above
3346 (define_insn "*neg_<shift>_si2_uxtw"
3347   [(set (match_operand:DI 0 "register_operand" "=r")
3348         (zero_extend:DI
3349          (neg:SI (ASHIFT:SI
3350                   (match_operand:SI 1 "register_operand" "r")
3351                   (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3352   ""
3353   "neg\\t%w0, %w1, <shift> %2"
3354   [(set_attr "type" "alu_shift_imm")]
3355 )
3356
3357 (define_insn "*neg_mul_imm_<mode>2"
3358   [(set (match_operand:GPI 0 "register_operand" "=r")
3359         (neg:GPI (mult:GPI
3360                   (match_operand:GPI 1 "register_operand" "r")
3361                   (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3362   ""
3363   "neg\\t%<w>0, %<w>1, lsl %p2"
3364   [(set_attr "type" "alu_shift_imm")]
3365 )
3366
3367 ;; zero_extend version of above
3368 (define_insn "*neg_mul_imm_si2_uxtw"
3369   [(set (match_operand:DI 0 "register_operand" "=r")
3370         (zero_extend:DI
3371          (neg:SI (mult:SI
3372                   (match_operand:SI 1 "register_operand" "r")
3373                   (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3374   ""
3375   "neg\\t%w0, %w1, lsl %p2"
3376   [(set_attr "type" "alu_shift_imm")]
3377 )
3378
3379 (define_insn "mul<mode>3"
3380   [(set (match_operand:GPI 0 "register_operand" "=r")
3381         (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3382                   (match_operand:GPI 2 "register_operand" "r")))]
3383   ""
3384   "mul\\t%<w>0, %<w>1, %<w>2"
3385   [(set_attr "type" "mul")]
3386 )
3387
3388 ;; zero_extend version of above
3389 (define_insn "*mulsi3_uxtw"
3390   [(set (match_operand:DI 0 "register_operand" "=r")
3391         (zero_extend:DI
3392          (mult:SI (match_operand:SI 1 "register_operand" "r")
3393                   (match_operand:SI 2 "register_operand" "r"))))]
3394   ""
3395   "mul\\t%w0, %w1, %w2"
3396   [(set_attr "type" "mul")]
3397 )
3398
3399 (define_insn "madd<mode>"
3400   [(set (match_operand:GPI 0 "register_operand" "=r")
3401         (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3402                             (match_operand:GPI 2 "register_operand" "r"))
3403                   (match_operand:GPI 3 "register_operand" "r")))]
3404   ""
3405   "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3406   [(set_attr "type" "mla")]
3407 )
3408
3409 ;; zero_extend version of above
3410 (define_insn "*maddsi_uxtw"
3411   [(set (match_operand:DI 0 "register_operand" "=r")
3412         (zero_extend:DI
3413          (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3414                            (match_operand:SI 2 "register_operand" "r"))
3415                   (match_operand:SI 3 "register_operand" "r"))))]
3416   ""
3417   "madd\\t%w0, %w1, %w2, %w3"
3418   [(set_attr "type" "mla")]
3419 )
3420
3421 (define_insn "*msub<mode>"
3422   [(set (match_operand:GPI 0 "register_operand" "=r")
3423         (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3424                    (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3425                              (match_operand:GPI 2 "register_operand" "r"))))]
3426
3427   ""
3428   "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3429   [(set_attr "type" "mla")]
3430 )
3431
3432 ;; zero_extend version of above
3433 (define_insn "*msubsi_uxtw"
3434   [(set (match_operand:DI 0 "register_operand" "=r")
3435         (zero_extend:DI
3436          (minus:SI (match_operand:SI 3 "register_operand" "r")
3437                    (mult:SI (match_operand:SI 1 "register_operand" "r")
3438                             (match_operand:SI 2 "register_operand" "r")))))]
3439
3440   ""
3441   "msub\\t%w0, %w1, %w2, %w3"
3442   [(set_attr "type" "mla")]
3443 )
3444
3445 (define_insn "*mul<mode>_neg"
3446   [(set (match_operand:GPI 0 "register_operand" "=r")
3447         (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3448                   (match_operand:GPI 2 "register_operand" "r")))]
3449
3450   ""
3451   "mneg\\t%<w>0, %<w>1, %<w>2"
3452   [(set_attr "type" "mul")]
3453 )
3454
3455 ;; zero_extend version of above
3456 (define_insn "*mulsi_neg_uxtw"
3457   [(set (match_operand:DI 0 "register_operand" "=r")
3458         (zero_extend:DI
3459          (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3460                   (match_operand:SI 2 "register_operand" "r"))))]
3461
3462   ""
3463   "mneg\\t%w0, %w1, %w2"
3464   [(set_attr "type" "mul")]
3465 )
3466
3467 (define_insn "<su_optab>mulsidi3"
3468   [(set (match_operand:DI 0 "register_operand" "=r")
3469         (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3470                  (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3471   ""
3472   "<su>mull\\t%0, %w1, %w2"
3473   [(set_attr "type" "<su>mull")]
3474 )
3475
3476 (define_insn "<su_optab>maddsidi4"
3477   [(set (match_operand:DI 0 "register_operand" "=r")
3478         (plus:DI (mult:DI
3479                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3480                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3481                  (match_operand:DI 3 "register_operand" "r")))]
3482   ""
3483   "<su>maddl\\t%0, %w1, %w2, %3"
3484   [(set_attr "type" "<su>mlal")]
3485 )
3486
3487 (define_insn "<su_optab>msubsidi4"
3488   [(set (match_operand:DI 0 "register_operand" "=r")
3489         (minus:DI
3490          (match_operand:DI 3 "register_operand" "r")
3491          (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3492                   (ANY_EXTEND:DI
3493                    (match_operand:SI 2 "register_operand" "r")))))]
3494   ""
3495   "<su>msubl\\t%0, %w1, %w2, %3"
3496   [(set_attr "type" "<su>mlal")]
3497 )
3498
3499 (define_insn "*<su_optab>mulsidi_neg"
3500   [(set (match_operand:DI 0 "register_operand" "=r")
3501         (mult:DI (neg:DI
3502                   (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3503                   (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3504   ""
3505   "<su>mnegl\\t%0, %w1, %w2"
3506   [(set_attr "type" "<su>mull")]
3507 )
3508
3509 (define_expand "<su_optab>mulditi3"
3510   [(set (match_operand:TI 0 "register_operand")
3511         (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3512                  (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3513   ""
3514 {
3515   rtx low = gen_reg_rtx (DImode);
3516   emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3517
3518   rtx high = gen_reg_rtx (DImode);
3519   emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3520
3521   emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3522   emit_move_insn (gen_highpart (DImode, operands[0]), high);
3523   DONE;
3524 })
3525
3526 ;; The default expansion of multi3 using umuldi3_highpart will perform
3527 ;; the additions in an order that fails to combine into two madd insns.
3528 (define_expand "multi3"
3529   [(set (match_operand:TI 0 "register_operand")
3530         (mult:TI (match_operand:TI 1 "register_operand")
3531                  (match_operand:TI 2 "register_operand")))]
3532   ""
3533 {
3534   rtx l0 = gen_reg_rtx (DImode);
3535   rtx l1 = gen_lowpart (DImode, operands[1]);
3536   rtx l2 = gen_lowpart (DImode, operands[2]);
3537   rtx h0 = gen_reg_rtx (DImode);
3538   rtx h1 = gen_highpart (DImode, operands[1]);
3539   rtx h2 = gen_highpart (DImode, operands[2]);
3540
3541   emit_insn (gen_muldi3 (l0, l1, l2));
3542   emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3543   emit_insn (gen_madddi (h0, h1, l2, h0));
3544   emit_insn (gen_madddi (h0, l1, h2, h0));
3545
3546   emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3547   emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3548   DONE;
3549 })
3550
3551 (define_insn "<su>muldi3_highpart"
3552   [(set (match_operand:DI 0 "register_operand" "=r")
3553         (truncate:DI
3554          (lshiftrt:TI
3555           (mult:TI
3556            (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3557            (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3558           (const_int 64))))]
3559   ""
3560   "<su>mulh\\t%0, %1, %2"
3561   [(set_attr "type" "<su>mull")]
3562 )
3563
3564 (define_insn "<su_optab>div<mode>3"
3565   [(set (match_operand:GPI 0 "register_operand" "=r")
3566         (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3567                      (match_operand:GPI 2 "register_operand" "r")))]
3568   ""
3569   "<su>div\\t%<w>0, %<w>1, %<w>2"
3570   [(set_attr "type" "<su>div")]
3571 )
3572
3573 ;; zero_extend version of above
3574 (define_insn "*<su_optab>divsi3_uxtw"
3575   [(set (match_operand:DI 0 "register_operand" "=r")
3576         (zero_extend:DI
3577          (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3578                      (match_operand:SI 2 "register_operand" "r"))))]
3579   ""
3580   "<su>div\\t%w0, %w1, %w2"
3581   [(set_attr "type" "<su>div")]
3582 )
3583
3584 ;; -------------------------------------------------------------------
3585 ;; Comparison insns
3586 ;; -------------------------------------------------------------------
3587
3588 (define_insn "cmp<mode>"
3589   [(set (reg:CC CC_REGNUM)
3590         (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3591                     (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3592   ""
3593   "@
3594    cmp\\t%<w>0, %<w>1
3595    cmp\\t%<w>0, %1
3596    cmn\\t%<w>0, #%n1"
3597   [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3598 )
3599
3600 (define_insn "fcmp<mode>"
3601   [(set (reg:CCFP CC_REGNUM)
3602         (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3603                       (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3604    "TARGET_FLOAT"
3605    "@
3606     fcmp\\t%<s>0, #0.0
3607     fcmp\\t%<s>0, %<s>1"
3608   [(set_attr "type" "fcmp<s>")]
3609 )
3610
3611 (define_insn "fcmpe<mode>"
3612   [(set (reg:CCFPE CC_REGNUM)
3613         (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3614                        (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3615    "TARGET_FLOAT"
3616    "@
3617     fcmpe\\t%<s>0, #0.0
3618     fcmpe\\t%<s>0, %<s>1"
3619   [(set_attr "type" "fcmp<s>")]
3620 )
3621
3622 (define_insn "*cmp_swp_<shift>_reg<mode>"
3623   [(set (reg:CC_SWP CC_REGNUM)
3624         (compare:CC_SWP (ASHIFT:GPI
3625                          (match_operand:GPI 0 "register_operand" "r")
3626                          (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3627                         (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3628   ""
3629   "cmp\\t%<w>2, %<w>0, <shift> %1"
3630   [(set_attr "type" "alus_shift_imm")]
3631 )
3632
3633 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3634   [(set (reg:CC_SWP CC_REGNUM)
3635         (compare:CC_SWP (ANY_EXTEND:GPI
3636                          (match_operand:ALLX 0 "register_operand" "r"))
3637                         (match_operand:GPI 1 "register_operand" "r")))]
3638   ""
3639   "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3640   [(set_attr "type" "alus_ext")]
3641 )
3642
3643 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3644   [(set (reg:CC_SWP CC_REGNUM)
3645         (compare:CC_SWP (ashift:GPI
3646                          (ANY_EXTEND:GPI
3647                           (match_operand:ALLX 0 "register_operand" "r"))
3648                          (match_operand 1 "aarch64_imm3" "Ui3"))
3649         (match_operand:GPI 2 "register_operand" "r")))]
3650   ""
3651   "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3652   [(set_attr "type" "alus_ext")]
3653 )
3654
3655 ;; -------------------------------------------------------------------
3656 ;; Store-flag and conditional select insns
3657 ;; -------------------------------------------------------------------
3658
3659 (define_expand "cstore<mode>4"
3660   [(set (match_operand:SI 0 "register_operand" "")
3661         (match_operator:SI 1 "aarch64_comparison_operator"
3662          [(match_operand:GPI 2 "register_operand" "")
3663           (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3664   ""
3665   "
3666   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3667                                       operands[3]);
3668   operands[3] = const0_rtx;
3669   "
3670 )
3671
3672 (define_expand "cstorecc4"
3673   [(set (match_operand:SI 0 "register_operand")
3674        (match_operator 1 "aarch64_comparison_operator_mode"
3675         [(match_operand 2 "cc_register")
3676          (match_operand 3 "const0_operand")]))]
3677   ""
3678 "{
3679   emit_insn (gen_rtx_SET (operands[0], operands[1]));
3680   DONE;
3681 }")
3682
3683
3684 (define_expand "cstore<mode>4"
3685   [(set (match_operand:SI 0 "register_operand" "")
3686         (match_operator:SI 1 "aarch64_comparison_operator_mode"
3687          [(match_operand:GPF 2 "register_operand" "")
3688           (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3689   ""
3690   "
3691   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3692                                       operands[3]);
3693   operands[3] = const0_rtx;
3694   "
3695 )
3696
3697 (define_insn "aarch64_cstore<mode>"
3698   [(set (match_operand:ALLI 0 "register_operand" "=r")
3699         (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3700          [(match_operand 2 "cc_register" "") (const_int 0)]))]
3701   ""
3702   "cset\\t%<w>0, %m1"
3703   [(set_attr "type" "csel")]
3704 )
3705
3706 ;; For a 24-bit immediate CST we can optimize the compare for equality
3707 ;; and branch sequence from:
3708 ;;      mov     x0, #imm1
3709 ;;      movk    x0, #imm2, lsl 16 /* x0 contains CST.  */
3710 ;;      cmp     x1, x0
3711 ;;      cset    x2, <ne,eq>
3712 ;; into the shorter:
3713 ;;      sub     x0, x1, #(CST & 0xfff000)
3714 ;;      subs    x0, x0, #(CST & 0x000fff)
3715 ;;      cset x2, <ne, eq>.
3716 (define_insn_and_split "*compare_cstore<mode>_insn"
3717   [(set (match_operand:GPI 0 "register_operand" "=r")
3718          (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3719                   (match_operand:GPI 2 "aarch64_imm24" "n")))
3720    (clobber (reg:CC CC_REGNUM))]
3721   "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3722    && !aarch64_plus_operand (operands[2], <MODE>mode)
3723    && !reload_completed"
3724   "#"
3725   "&& true"
3726   [(const_int 0)]
3727   {
3728     HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3729     HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3730     rtx tmp = gen_reg_rtx (<MODE>mode);
3731     emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3732     emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3733     rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3734     rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3735     emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3736     DONE;
3737   }
3738   [(set_attr "type" "csel")]
3739 )
3740
3741 ;; zero_extend version of the above
3742 (define_insn "*cstoresi_insn_uxtw"
3743   [(set (match_operand:DI 0 "register_operand" "=r")
3744         (zero_extend:DI
3745          (match_operator:SI 1 "aarch64_comparison_operator_mode"
3746           [(match_operand 2 "cc_register" "") (const_int 0)])))]
3747   ""
3748   "cset\\t%w0, %m1"
3749   [(set_attr "type" "csel")]
3750 )
3751
3752 (define_insn "cstore<mode>_neg"
3753   [(set (match_operand:ALLI 0 "register_operand" "=r")
3754         (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3755                   [(match_operand 2 "cc_register" "") (const_int 0)])))]
3756   ""
3757   "csetm\\t%<w>0, %m1"
3758   [(set_attr "type" "csel")]
3759 )
3760
3761 ;; zero_extend version of the above
3762 (define_insn "*cstoresi_neg_uxtw"
3763   [(set (match_operand:DI 0 "register_operand" "=r")
3764         (zero_extend:DI
3765          (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3766                   [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3767   ""
3768   "csetm\\t%w0, %m1"
3769   [(set_attr "type" "csel")]
3770 )
3771
3772 (define_expand "cmov<mode>6"
3773   [(set (match_operand:GPI 0 "register_operand" "")
3774         (if_then_else:GPI
3775          (match_operator 1 "aarch64_comparison_operator"
3776           [(match_operand:GPI 2 "register_operand" "")
3777            (match_operand:GPI 3 "aarch64_plus_operand" "")])
3778          (match_operand:GPI 4 "register_operand" "")
3779          (match_operand:GPI 5 "register_operand" "")))]
3780   ""
3781   "
3782   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3783                                       operands[3]);
3784   operands[3] = const0_rtx;
3785   "
3786 )
3787
3788 (define_expand "cmov<mode>6"
3789   [(set (match_operand:GPF 0 "register_operand" "")
3790         (if_then_else:GPF
3791          (match_operator 1 "aarch64_comparison_operator"
3792           [(match_operand:GPF 2 "register_operand" "")
3793            (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3794          (match_operand:GPF 4 "register_operand" "")
3795          (match_operand:GPF 5 "register_operand" "")))]
3796   ""
3797   "
3798   operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3799                                       operands[3]);
3800   operands[3] = const0_rtx;
3801   "
3802 )
3803
3804 (define_insn "*cmov<mode>_insn"
3805   [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3806         (if_then_else:ALLI
3807          (match_operator 1 "aarch64_comparison_operator"
3808           [(match_operand 2 "cc_register" "") (const_int 0)])
3809          (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3810          (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3811   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3812      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3813   ;; Final two alternatives should be unreachable, but included for completeness
3814   "@
3815    csel\\t%<w>0, %<w>3, %<w>4, %m1
3816    csinv\\t%<w>0, %<w>3, <w>zr, %m1
3817    csinv\\t%<w>0, %<w>4, <w>zr, %M1
3818    csinc\\t%<w>0, %<w>3, <w>zr, %m1
3819    csinc\\t%<w>0, %<w>4, <w>zr, %M1
3820    mov\\t%<w>0, -1
3821    mov\\t%<w>0, 1"
3822   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3823 )
3824
3825 ;; zero_extend version of above
3826 (define_insn "*cmovsi_insn_uxtw"
3827   [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3828         (zero_extend:DI
3829          (if_then_else:SI
3830           (match_operator 1 "aarch64_comparison_operator"
3831            [(match_operand 2 "cc_register" "") (const_int 0)])
3832           (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3833           (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3834   "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3835      || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3836   ;; Final two alternatives should be unreachable, but included for completeness
3837   "@
3838    csel\\t%w0, %w3, %w4, %m1
3839    csinv\\t%w0, %w3, wzr, %m1
3840    csinv\\t%w0, %w4, wzr, %M1
3841    csinc\\t%w0, %w3, wzr, %m1
3842    csinc\\t%w0, %w4, wzr, %M1
3843    mov\\t%w0, -1
3844    mov\\t%w0, 1"
3845   [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3846 )
3847
3848 (define_insn "*cmovdi_insn_uxtw"
3849   [(set (match_operand:DI 0 "register_operand" "=r")
3850         (if_then_else:DI
3851          (match_operator 1 "aarch64_comparison_operator"
3852           [(match_operand 2 "cc_register" "") (const_int 0)])
3853          (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3854          (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3855   ""
3856   "csel\\t%w0, %w3, %w4, %m1"
3857   [(set_attr "type" "csel")]
3858 )
3859
3860 (define_insn "*cmov<mode>_insn"
3861   [(set (match_operand:GPF 0 "register_operand" "=w")
3862         (if_then_else:GPF
3863          (match_operator 1 "aarch64_comparison_operator"
3864           [(match_operand 2 "cc_register" "") (const_int 0)])
3865          (match_operand:GPF 3 "register_operand" "w")
3866          (match_operand:GPF 4 "register_operand" "w")))]
3867   "TARGET_FLOAT"
3868   "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3869   [(set_attr "type" "fcsel")]
3870 )
3871
3872 (define_expand "mov<mode>cc"
3873   [(set (match_operand:ALLI 0 "register_operand" "")
3874         (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3875                            (match_operand:ALLI 2 "register_operand" "")
3876                            (match_operand:ALLI 3 "register_operand" "")))]
3877   ""
3878   {
3879     rtx ccreg;
3880     enum rtx_code code = GET_CODE (operands[1]);
3881
3882     if (code == UNEQ || code == LTGT)
3883       FAIL;
3884
3885     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3886                                      XEXP (operands[1], 1));
3887     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3888   }
3889 )
3890
3891 (define_expand "mov<GPF:mode><GPI:mode>cc"
3892   [(set (match_operand:GPI 0 "register_operand" "")
3893         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3894                           (match_operand:GPF 2 "register_operand" "")
3895                           (match_operand:GPF 3 "register_operand" "")))]
3896   ""
3897   {
3898     rtx ccreg;
3899     enum rtx_code code = GET_CODE (operands[1]);
3900
3901     if (code == UNEQ || code == LTGT)
3902       FAIL;
3903
3904     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3905                                   XEXP (operands[1], 1));
3906     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3907   }
3908 )
3909
3910 (define_expand "mov<mode>cc"
3911   [(set (match_operand:GPF 0 "register_operand" "")
3912         (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3913                           (match_operand:GPF 2 "register_operand" "")
3914                           (match_operand:GPF 3 "register_operand" "")))]
3915   ""
3916   {
3917     rtx ccreg;
3918     enum rtx_code code = GET_CODE (operands[1]);
3919
3920     if (code == UNEQ || code == LTGT)
3921       FAIL;
3922
3923     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3924                                   XEXP (operands[1], 1));
3925     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3926   }
3927 )
3928
3929 (define_expand "<neg_not_op><mode>cc"
3930   [(set (match_operand:GPI 0 "register_operand" "")
3931         (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3932                           (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3933                           (match_operand:GPI 3 "register_operand" "")))]
3934   ""
3935   {
3936     rtx ccreg;
3937     enum rtx_code code = GET_CODE (operands[1]);
3938
3939     if (code == UNEQ || code == LTGT)
3940       FAIL;
3941
3942     ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3943                                       XEXP (operands[1], 1));
3944     operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3945   }
3946 )
3947
3948 ;; CRC32 instructions.
3949 (define_insn "aarch64_<crc_variant>"
3950   [(set (match_operand:SI 0 "register_operand" "=r")
3951         (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3952                     (match_operand:<crc_mode> 2 "register_operand" "r")]
3953          CRC))]
3954   "TARGET_CRC32"
3955   {
3956     if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
3957       return "<crc_variant>\\t%w0, %w1, %x2";
3958     else
3959       return "<crc_variant>\\t%w0, %w1, %w2";
3960   }
3961   [(set_attr "type" "crc")]
3962 )
3963
3964 (define_insn "*csinc2<mode>_insn"
3965   [(set (match_operand:GPI 0 "register_operand" "=r")
3966         (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3967                   (match_operand:GPI 1 "register_operand" "r")))]
3968   ""
3969   "cinc\\t%<w>0, %<w>1, %m2"
3970   [(set_attr "type" "csel")]
3971 )
3972
3973 (define_insn "csinc3<mode>_insn"
3974   [(set (match_operand:GPI 0 "register_operand" "=r")
3975         (if_then_else:GPI
3976           (match_operand 1 "aarch64_comparison_operation" "")
3977           (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3978                     (const_int 1))
3979           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3980   ""
3981   "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3982   [(set_attr "type" "csel")]
3983 )
3984
3985 (define_insn "*csinv3<mode>_insn"
3986   [(set (match_operand:GPI 0 "register_operand" "=r")
3987         (if_then_else:GPI
3988           (match_operand 1 "aarch64_comparison_operation" "")
3989           (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3990           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3991   ""
3992   "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3993   [(set_attr "type" "csel")]
3994 )
3995
3996 (define_insn "csneg3_uxtw_insn"
3997   [(set (match_operand:DI 0 "register_operand" "=r")
3998         (zero_extend:DI
3999           (if_then_else:SI
4000             (match_operand 1 "aarch64_comparison_operation" "")
4001             (neg:SI (match_operand:SI 2 "register_operand" "r"))
4002             (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
4003   ""
4004   "csneg\\t%w0, %w3, %w2, %M1"
4005   [(set_attr "type" "csel")]
4006 )
4007
4008 (define_insn "csneg3<mode>_insn"
4009   [(set (match_operand:GPI 0 "register_operand" "=r")
4010         (if_then_else:GPI
4011           (match_operand 1 "aarch64_comparison_operation" "")
4012           (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4013           (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4014   ""
4015   "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4016   [(set_attr "type" "csel")]
4017 )
4018
4019 ;; If X can be loaded by a single CNT[BHWD] instruction,
4020 ;;
4021 ;;    A = UMAX (B, X)
4022 ;;
4023 ;; is equivalent to:
4024 ;;
4025 ;;    TMP = UQDEC[BHWD] (B, X)
4026 ;;    A = TMP + X
4027 ;;
4028 ;; Defining the pattern this way means that:
4029 ;;
4030 ;;    A = UMAX (B, X) - X
4031 ;;
4032 ;; becomes:
4033 ;;
4034 ;;    TMP1 = UQDEC[BHWD] (B, X)
4035 ;;    TMP2 = TMP1 + X
4036 ;;    A = TMP2 - X
4037 ;;
4038 ;; which combine can optimize to:
4039 ;;
4040 ;;    A = UQDEC[BHWD] (B, X)
4041 ;;
4042 ;; We don't use match_operand predicates because the order of the operands
4043 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4044 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4045 (define_expand "umax<mode>3"
4046   [(set (match_operand:GPI 0 "register_operand")
4047         (umax:GPI (match_operand:GPI 1 "")
4048                   (match_operand:GPI 2 "")))]
4049   "TARGET_SVE"
4050   {
4051     if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4052       std::swap (operands[1], operands[2]);
4053     else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4054       FAIL;
4055     rtx temp = gen_reg_rtx (<MODE>mode);
4056     operands[1] = force_reg (<MODE>mode, operands[1]);
4057     emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4058     emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4059     DONE;
4060   }
4061 )
4062
4063 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4064 (define_insn "aarch64_uqdec<mode>"
4065   [(set (match_operand:GPI 0 "register_operand" "=r")
4066         (minus:GPI
4067          (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4068                    (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4069          (match_dup 2)))]
4070   "TARGET_SVE"
4071   {
4072     return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4073   }
4074 )
4075
4076 ;; -------------------------------------------------------------------
4077 ;; Logical operations
4078 ;; -------------------------------------------------------------------
4079
4080
4081 (define_insn_and_split "*aarch64_and<mode>_imm2"
4082   [(set (match_operand:GPI 0 "register_operand" "=rk")
4083         (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4084                  (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4085   ""
4086   "#"
4087   "true"
4088   [(const_int 0)]
4089   {
4090      HOST_WIDE_INT val = INTVAL (operands[2]);
4091      rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4092      rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4093
4094      emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4095      emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4096      DONE;
4097   }
4098 )
4099
4100 (define_insn "<optab><mode>3"
4101   [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4102         (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4103                      (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4104   ""
4105   "@
4106   <logical>\\t%<w>0, %<w>1, %<w>2
4107   <logical>\\t%<w>0, %<w>1, %2
4108   <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4109   [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4110    (set_attr "arch" "*,*,simd")]
4111 )
4112
4113 ;; zero_extend version of above
4114 (define_insn "*<optab>si3_uxtw"
4115   [(set (match_operand:DI 0 "register_operand" "=r,rk")
4116         (zero_extend:DI
4117          (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4118                      (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4119   ""
4120   "@
4121    <logical>\\t%w0, %w1, %w2
4122    <logical>\\t%w0, %w1, %2"
4123   [(set_attr "type" "logic_reg,logic_imm")]
4124 )
4125
4126 (define_insn "*and<mode>3_compare0"
4127   [(set (reg:CC_NZ CC_REGNUM)
4128         (compare:CC_NZ
4129          (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4130                   (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4131          (const_int 0)))
4132    (set (match_operand:GPI 0 "register_operand" "=r,r")
4133         (and:GPI (match_dup 1) (match_dup 2)))]
4134   ""
4135   "@
4136    ands\\t%<w>0, %<w>1, %<w>2
4137    ands\\t%<w>0, %<w>1, %2"
4138   [(set_attr "type" "logics_reg,logics_imm")]
4139 )
4140
4141 ;; zero_extend version of above
4142 (define_insn "*andsi3_compare0_uxtw"
4143   [(set (reg:CC_NZ CC_REGNUM)
4144         (compare:CC_NZ
4145          (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4146                  (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4147          (const_int 0)))
4148    (set (match_operand:DI 0 "register_operand" "=r,r")
4149         (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4150   ""
4151   "@
4152    ands\\t%w0, %w1, %w2
4153    ands\\t%w0, %w1, %2"
4154   [(set_attr "type" "logics_reg,logics_imm")]
4155 )
4156
4157 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4158   [(set (reg:CC_NZ CC_REGNUM)
4159         (compare:CC_NZ
4160          (and:GPI (SHIFT:GPI
4161                    (match_operand:GPI 1 "register_operand" "r")
4162                    (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4163                   (match_operand:GPI 3 "register_operand" "r"))
4164          (const_int 0)))
4165    (set (match_operand:GPI 0 "register_operand" "=r")
4166         (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4167   ""
4168   "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4169   [(set_attr "type" "logics_shift_imm")]
4170 )
4171
4172 ;; zero_extend version of above
4173 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4174   [(set (reg:CC_NZ CC_REGNUM)
4175         (compare:CC_NZ
4176          (and:SI (SHIFT:SI
4177                   (match_operand:SI 1 "register_operand" "r")
4178                   (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4179                  (match_operand:SI 3 "register_operand" "r"))
4180          (const_int 0)))
4181    (set (match_operand:DI 0 "register_operand" "=r")
4182         (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4183                                 (match_dup 3))))]
4184   ""
4185   "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4186   [(set_attr "type" "logics_shift_imm")]
4187 )
4188
4189 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4190   [(set (match_operand:GPI 0 "register_operand" "=r")
4191         (LOGICAL:GPI (SHIFT:GPI
4192                       (match_operand:GPI 1 "register_operand" "r")
4193                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4194                      (match_operand:GPI 3 "register_operand" "r")))]
4195   ""
4196   "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4197   [(set_attr "type" "logic_shift_imm")]
4198 )
4199
4200 (define_insn "*<optab>_rol<mode>3"
4201   [(set (match_operand:GPI 0 "register_operand" "=r")
4202         (LOGICAL:GPI (rotate:GPI
4203                       (match_operand:GPI 1 "register_operand" "r")
4204                       (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4205                      (match_operand:GPI 3 "register_operand" "r")))]
4206   ""
4207   "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4208   [(set_attr "type" "logic_shift_imm")]
4209 )
4210
4211 ;; zero_extend versions of above
4212 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4213   [(set (match_operand:DI 0 "register_operand" "=r")
4214         (zero_extend:DI
4215          (LOGICAL:SI (SHIFT:SI
4216                       (match_operand:SI 1 "register_operand" "r")
4217                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4218                      (match_operand:SI 3 "register_operand" "r"))))]
4219   ""
4220   "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4221   [(set_attr "type" "logic_shift_imm")]
4222 )
4223
4224 (define_insn "*<optab>_rolsi3_uxtw"
4225   [(set (match_operand:DI 0 "register_operand" "=r")
4226         (zero_extend:DI
4227          (LOGICAL:SI (rotate:SI
4228                       (match_operand:SI 1 "register_operand" "r")
4229                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4230                      (match_operand:SI 3 "register_operand" "r"))))]
4231   ""
4232   "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4233   [(set_attr "type" "logic_shift_imm")]
4234 )
4235
4236 (define_insn "one_cmpl<mode>2"
4237   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4238         (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4239   ""
4240   "@
4241   mvn\\t%<w>0, %<w>1
4242   mvn\\t%0.8b, %1.8b"
4243   [(set_attr "type" "logic_reg,neon_logic")
4244    (set_attr "arch" "*,simd")]
4245 )
4246
4247 (define_insn "*one_cmpl_<optab><mode>2"
4248   [(set (match_operand:GPI 0 "register_operand" "=r")
4249         (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4250                             (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4251   ""
4252   "mvn\\t%<w>0, %<w>1, <shift> %2"
4253   [(set_attr "type" "logic_shift_imm")]
4254 )
4255
4256 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4257
4258 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4259   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4260         (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4261                      (match_operand:GPI 2 "register_operand" "r,w")))]
4262   ""
4263   "@
4264   <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4265   <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4266   [(set_attr "type" "logic_reg,neon_logic")
4267    (set_attr "arch" "*,simd")]
4268 )
4269
4270 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4271   [(set (match_operand:DI 0 "register_operand" "=r")
4272         (zero_extend:DI
4273           (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4274                        (match_operand:SI 2 "register_operand" "r"))))]
4275   ""
4276   "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4277   [(set_attr "type" "logic_reg")]
4278 )
4279
4280 (define_insn "*xor_one_cmplsidi3_ze"
4281   [(set (match_operand:DI 0 "register_operand" "=r")
4282         (zero_extend:DI
4283           (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4284                           (match_operand:SI 2 "register_operand" "r")))))]
4285   ""
4286   "eon\\t%w0, %w1, %w2"
4287   [(set_attr "type" "logic_reg")]
4288 )
4289
4290 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4291 ;; eon does not operate on SIMD registers so the vector variant must be split.
4292 (define_insn_and_split "*xor_one_cmpl<mode>3"
4293   [(set (match_operand:GPI 0 "register_operand" "=r,w")
4294         (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4295                           (match_operand:GPI 2 "register_operand" "r,w"))))]
4296   ""
4297   "@
4298   eon\\t%<w>0, %<w>1, %<w>2
4299   #"
4300   "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4301   [(set (match_operand:GPI 0 "register_operand" "=w")
4302         (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4303                  (match_operand:GPI 2 "register_operand" "w")))
4304    (set (match_dup 0) (not:GPI (match_dup 0)))]
4305   ""
4306   [(set_attr "type" "logic_reg,multiple")
4307    (set_attr "arch" "*,simd")]
4308 )
4309
4310 (define_insn "*and_one_cmpl<mode>3_compare0"
4311   [(set (reg:CC_NZ CC_REGNUM)
4312         (compare:CC_NZ
4313          (and:GPI (not:GPI
4314                    (match_operand:GPI 1 "register_operand" "r"))
4315                   (match_operand:GPI 2 "register_operand" "r"))
4316          (const_int 0)))
4317    (set (match_operand:GPI 0 "register_operand" "=r")
4318         (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4319   ""
4320   "bics\\t%<w>0, %<w>2, %<w>1"
4321   [(set_attr "type" "logics_reg")]
4322 )
4323
4324 ;; zero_extend version of above
4325 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4326   [(set (reg:CC_NZ CC_REGNUM)
4327         (compare:CC_NZ
4328          (and:SI (not:SI
4329                   (match_operand:SI 1 "register_operand" "r"))
4330                  (match_operand:SI 2 "register_operand" "r"))
4331          (const_int 0)))
4332    (set (match_operand:DI 0 "register_operand" "=r")
4333         (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4334   ""
4335   "bics\\t%w0, %w2, %w1"
4336   [(set_attr "type" "logics_reg")]
4337 )
4338
4339 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4340   [(set (reg:CC_NZ CC_REGNUM)
4341     (compare:CC_NZ
4342      (and:GPI (not:GPI
4343            (match_operand:GPI 0 "register_operand" "r"))
4344           (match_operand:GPI 1 "register_operand" "r"))
4345      (const_int 0)))]
4346   ""
4347   "bics\\t<w>zr, %<w>1, %<w>0"
4348   [(set_attr "type" "logics_reg")]
4349 )
4350
4351 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4352   [(set (match_operand:GPI 0 "register_operand" "=r")
4353         (LOGICAL:GPI (not:GPI
4354                       (SHIFT:GPI
4355                        (match_operand:GPI 1 "register_operand" "r")
4356                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4357                      (match_operand:GPI 3 "register_operand" "r")))]
4358   ""
4359   "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4360   [(set_attr "type" "logic_shift_imm")]
4361 )
4362
4363 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4364   [(set (match_operand:GPI 0 "register_operand" "=r")
4365         (not:GPI (xor:GPI
4366                       (SHIFT:GPI
4367                        (match_operand:GPI 1 "register_operand" "r")
4368                        (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4369                      (match_operand:GPI 3 "register_operand" "r"))))]
4370   ""
4371   "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4372   [(set_attr "type" "logic_shift_imm")]
4373 )
4374
4375 ;; Zero-extend version of the above.
4376 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4377   [(set (match_operand:DI 0 "register_operand" "=r")
4378         (zero_extend:DI
4379           (not:SI (xor:SI
4380                     (SHIFT:SI
4381                       (match_operand:SI 1 "register_operand" "r")
4382                       (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4383                     (match_operand:SI 3 "register_operand" "r")))))]
4384   ""
4385   "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4386   [(set_attr "type" "logic_shift_imm")]
4387 )
4388
4389 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4390   [(set (reg:CC_NZ CC_REGNUM)
4391         (compare:CC_NZ
4392          (and:GPI (not:GPI
4393                    (SHIFT:GPI
4394                     (match_operand:GPI 1 "register_operand" "r")
4395                     (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4396                   (match_operand:GPI 3 "register_operand" "r"))
4397          (const_int 0)))
4398    (set (match_operand:GPI 0 "register_operand" "=r")
4399         (and:GPI (not:GPI
4400                   (SHIFT:GPI
4401                    (match_dup 1) (match_dup 2))) (match_dup 3)))]
4402   ""
4403   "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4404   [(set_attr "type" "logics_shift_imm")]
4405 )
4406
4407 ;; zero_extend version of above
4408 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4409   [(set (reg:CC_NZ CC_REGNUM)
4410         (compare:CC_NZ
4411          (and:SI (not:SI
4412                   (SHIFT:SI
4413                    (match_operand:SI 1 "register_operand" "r")
4414                    (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4415                  (match_operand:SI 3 "register_operand" "r"))
4416          (const_int 0)))
4417    (set (match_operand:DI 0 "register_operand" "=r")
4418         (zero_extend:DI (and:SI
4419                          (not:SI
4420                           (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4421   ""
4422   "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4423   [(set_attr "type" "logics_shift_imm")]
4424 )
4425
4426 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4427   [(set (reg:CC_NZ CC_REGNUM)
4428     (compare:CC_NZ
4429      (and:GPI (not:GPI
4430            (SHIFT:GPI
4431             (match_operand:GPI 0 "register_operand" "r")
4432             (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4433           (match_operand:GPI 2 "register_operand" "r"))
4434      (const_int 0)))]
4435   ""
4436   "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4437   [(set_attr "type" "logics_shift_imm")]
4438 )
4439
4440 (define_insn "clz<mode>2"
4441   [(set (match_operand:GPI 0 "register_operand" "=r")
4442         (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4443   ""
4444   "clz\\t%<w>0, %<w>1"
4445   [(set_attr "type" "clz")]
4446 )
4447
4448 (define_expand "ffs<mode>2"
4449   [(match_operand:GPI 0 "register_operand")
4450    (match_operand:GPI 1 "register_operand")]
4451   ""
4452   {
4453     rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4454     rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4455
4456     emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4457     emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4458     emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4459     DONE;
4460   }
4461 )
4462
4463 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4464 ;;
4465 ;; MOV  v.1d, x0
4466 ;; CNT  v1.8b, v.8b
4467 ;; ADDV b2, v1.8b
4468 ;; MOV  w0, v2.b[0]
4469
4470 (define_expand "popcount<mode>2"
4471   [(match_operand:GPI 0 "register_operand")
4472    (match_operand:GPI 1 "register_operand")]
4473   "TARGET_SIMD"
4474 {
4475   rtx v = gen_reg_rtx (V8QImode);
4476   rtx v1 = gen_reg_rtx (V8QImode);
4477   rtx r = gen_reg_rtx (QImode);
4478   rtx in = operands[1];
4479   rtx out = operands[0];
4480   if(<MODE>mode == SImode)
4481     {
4482       rtx tmp;
4483       tmp = gen_reg_rtx (DImode);
4484       /* If we have SImode, zero extend to DImode, pop count does
4485          not change if we have extra zeros. */
4486       emit_insn (gen_zero_extendsidi2 (tmp, in));
4487       in = tmp;
4488     }
4489   emit_move_insn (v, gen_lowpart (V8QImode, in));
4490   emit_insn (gen_popcountv8qi2 (v1, v));
4491   emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4492   emit_insn (gen_zero_extendqi<mode>2 (out, r));
4493   DONE;
4494 })
4495
4496 (define_insn "clrsb<mode>2"
4497   [(set (match_operand:GPI 0 "register_operand" "=r")
4498         (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4499   ""
4500   "cls\\t%<w>0, %<w>1"
4501   [(set_attr "type" "clz")]
4502 )
4503
4504 (define_insn "rbit<mode>2"
4505   [(set (match_operand:GPI 0 "register_operand" "=r")
4506         (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4507   ""
4508   "rbit\\t%<w>0, %<w>1"
4509   [(set_attr "type" "rbit")]
4510 )
4511
4512 ;; Split after reload into RBIT + CLZ.  Since RBIT is represented as an UNSPEC
4513 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4514 ;; expression and split after reload to enable scheduling them apart if
4515 ;; needed.
4516
4517 (define_insn_and_split "ctz<mode>2"
4518  [(set (match_operand:GPI           0 "register_operand" "=r")
4519        (ctz:GPI (match_operand:GPI  1 "register_operand" "r")))]
4520   ""
4521   "#"
4522   "reload_completed"
4523   [(const_int 0)]
4524   "
4525   emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4526   emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4527   DONE;
4528 ")
4529
4530 (define_insn "*and<mode>_compare0"
4531   [(set (reg:CC_NZ CC_REGNUM)
4532         (compare:CC_NZ
4533          (match_operand:SHORT 0 "register_operand" "r")
4534          (const_int 0)))]
4535   ""
4536   "tst\\t%<w>0, <short_mask>"
4537   [(set_attr "type" "alus_imm")]
4538 )
4539
4540 (define_insn "*ands<mode>_compare0"
4541   [(set (reg:CC_NZ CC_REGNUM)
4542         (compare:CC_NZ
4543          (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4544          (const_int 0)))
4545    (set (match_operand:GPI 0 "register_operand" "=r")
4546         (zero_extend:GPI (match_dup 1)))]
4547   ""
4548   "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4549   [(set_attr "type" "alus_imm")]
4550 )
4551
4552 (define_insn "*and<mode>3nr_compare0"
4553   [(set (reg:CC_NZ CC_REGNUM)
4554         (compare:CC_NZ
4555          (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4556                   (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4557          (const_int 0)))]
4558   ""
4559   "@
4560    tst\\t%<w>0, %<w>1
4561    tst\\t%<w>0, %1"
4562   [(set_attr "type" "logics_reg,logics_imm")]
4563 )
4564
4565 (define_split
4566   [(set (reg:CC_NZ CC_REGNUM)
4567         (compare:CC_NZ
4568          (and:GPI (match_operand:GPI 0 "register_operand")
4569                   (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4570          (const_int 0)))
4571    (clobber (match_operand:SI 2 "register_operand"))]
4572   ""
4573   [(set (match_dup 2) (match_dup 1))
4574    (set (reg:CC_NZ CC_REGNUM)
4575         (compare:CC_NZ
4576          (and:GPI (match_dup 0)
4577                   (match_dup 2))
4578          (const_int 0)))]
4579 )
4580
4581 (define_insn "*and<mode>3nr_compare0_zextract"
4582   [(set (reg:CC_NZ CC_REGNUM)
4583         (compare:CC_NZ
4584          (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4585                   (match_operand:GPI 1 "const_int_operand" "n")
4586                   (match_operand:GPI 2 "const_int_operand" "n"))
4587          (const_int 0)))]
4588   "INTVAL (operands[1]) > 0
4589    && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4590         <= GET_MODE_BITSIZE (<MODE>mode))
4591    && aarch64_bitmask_imm (
4592         UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4593                                                  operands[2])),
4594         <MODE>mode)"
4595   {
4596     operands[1]
4597       = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4598     return "tst\\t%<w>0, %1";
4599   }
4600   [(set_attr "type" "logics_shift_imm")]
4601 )
4602
4603 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4604   [(set (reg:CC_NZ CC_REGNUM)
4605         (compare:CC_NZ
4606          (and:GPI (SHIFT:GPI
4607                    (match_operand:GPI 0 "register_operand" "r")
4608                    (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4609                   (match_operand:GPI 2 "register_operand" "r"))
4610         (const_int 0)))]
4611   ""
4612   "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4613   [(set_attr "type" "logics_shift_imm")]
4614 )
4615
4616 (define_split
4617   [(set (reg:CC_NZ CC_REGNUM)
4618         (compare:CC_NZ
4619          (and:GPI (SHIFT:GPI
4620                    (match_operand:GPI 0 "register_operand")
4621                    (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4622                   (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4623         (const_int 0)))
4624     (clobber (match_operand:SI 3 "register_operand"))]
4625   ""
4626   [(set (match_dup 3) (match_dup 2))
4627    (set (reg:CC_NZ CC_REGNUM)
4628         (compare:CC_NZ
4629          (and:GPI (SHIFT:GPI
4630                    (match_dup 0)
4631                    (match_dup 1))
4632                   (match_dup 3))
4633          (const_int 0)))]
4634 )
4635
4636 ;; -------------------------------------------------------------------
4637 ;; Shifts
4638 ;; -------------------------------------------------------------------
4639
4640 (define_expand "<optab><mode>3"
4641   [(set (match_operand:GPI 0 "register_operand")
4642         (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4643                     (match_operand:QI 2 "aarch64_reg_or_imm")))]
4644   ""
4645   {
4646     if (CONST_INT_P (operands[2]))
4647       {
4648         operands[2] = GEN_INT (INTVAL (operands[2])
4649                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4650
4651         if (operands[2] == const0_rtx)
4652           {
4653             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4654             DONE;
4655           }
4656       }
4657   }
4658 )
4659
4660 (define_expand "ashl<mode>3"
4661   [(set (match_operand:SHORT 0 "register_operand")
4662         (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4663                       (match_operand:QI 2 "const_int_operand")))]
4664   ""
4665   {
4666     operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4667
4668     if (operands[2] == const0_rtx)
4669       {
4670         emit_insn (gen_mov<mode> (operands[0], operands[1]));
4671         DONE;
4672       }
4673   }
4674 )
4675
4676 (define_expand "rotr<mode>3"
4677   [(set (match_operand:GPI 0 "register_operand")
4678         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4679                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4680   ""
4681   {
4682     if (CONST_INT_P (operands[2]))
4683       {
4684         operands[2] = GEN_INT (INTVAL (operands[2])
4685                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4686
4687         if (operands[2] == const0_rtx)
4688           {
4689             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4690             DONE;
4691           }
4692       }
4693   }
4694 )
4695
4696 (define_expand "rotl<mode>3"
4697   [(set (match_operand:GPI 0 "register_operand")
4698         (rotatert:GPI (match_operand:GPI 1 "register_operand")
4699                       (match_operand:QI 2 "aarch64_reg_or_imm")))]
4700   ""
4701   {
4702     /* (SZ - cnt) % SZ == -cnt % SZ */
4703     if (CONST_INT_P (operands[2]))
4704       {
4705         operands[2] = GEN_INT ((-INTVAL (operands[2]))
4706                                & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4707         if (operands[2] == const0_rtx)
4708           {
4709             emit_insn (gen_mov<mode> (operands[0], operands[1]));
4710             DONE;
4711           }
4712       }
4713     else
4714       operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4715                                         NULL_RTX, 1);
4716   }
4717 )
4718
4719 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4720 ;; they truncate the shift/rotate amount by the size of the registers they
4721 ;; operate on: 32 for W-regs, 64 for X-regs.  This allows us to optimise away
4722 ;; such redundant masking instructions.  GCC can do that automatically when
4723 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4724 ;; because some of the SISD shift alternatives don't perform this truncations.
4725 ;; So this pattern exists to catch such cases.
4726
4727 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4728   [(set (match_operand:GPI 0 "register_operand" "=r")
4729         (SHIFT:GPI
4730           (match_operand:GPI 1 "register_operand" "r")
4731           (match_operator 4 "subreg_lowpart_operator"
4732            [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4733                      (match_operand 3 "const_int_operand" "n"))])))]
4734   "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4735   "<shift>\t%<w>0, %<w>1, %<w>2"
4736   [(set_attr "type" "shift_reg")]
4737 )
4738
4739 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4740   [(set (match_operand:GPI 0 "register_operand" "=&r")
4741         (SHIFT:GPI
4742           (match_operand:GPI 1 "register_operand" "r")
4743           (match_operator 4 "subreg_lowpart_operator"
4744           [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4745                            (match_operand 3 "const_int_operand" "n")))])))]
4746   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4747   "#"
4748   "&& true"
4749   [(const_int 0)]
4750   {
4751     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4752                : lowpart_subreg (SImode, operands[0], <MODE>mode));
4753     emit_insn (gen_negsi2 (tmp, operands[2]));
4754
4755     rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4756     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4757                                      SUBREG_BYTE (operands[4]));
4758     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4759     DONE;
4760   }
4761 )
4762
4763 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
4764   [(set (match_operand:GPI 0 "register_operand" "=&r")
4765         (ashift:GPI
4766           (match_operand:GPI 1 "register_operand" "r")
4767           (minus:QI (match_operand 2 "const_int_operand" "n")
4768                     (match_operator 5 "subreg_lowpart_operator"
4769                     [(and:SI (match_operand:SI 3 "register_operand" "r")
4770                              (match_operand 4 "const_int_operand" "n"))]))))]
4771   "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4772    && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4773   "#"
4774   "&& true"
4775   [(const_int 0)]
4776   {
4777     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4778                : operands[0]);
4779
4780     emit_insn (gen_negsi2 (tmp, operands[3]));
4781
4782     rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
4783     rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
4784                                      SUBREG_BYTE (operands[5]));
4785
4786     emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
4787     DONE;
4788   }
4789 )
4790
4791 (define_insn "*aarch64_<optab>_reg_di3_mask2"
4792   [(set (match_operand:DI 0 "register_operand" "=r")
4793         (SHIFT:DI
4794           (match_operand:DI 1 "register_operand" "r")
4795           (match_operator 4 "subreg_lowpart_operator"
4796            [(and:SI (match_operand:SI 2 "register_operand" "r")
4797                     (match_operand 3 "const_int_operand" "n"))])))]
4798   "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
4799 {
4800   rtx xop[3];
4801   xop[0] = operands[0];
4802   xop[1] = operands[1];
4803   xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
4804   output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
4805   return "";
4806 }
4807   [(set_attr "type" "shift_reg")]
4808 )
4809
4810 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
4811   [(set (match_operand:GPI 0 "register_operand" "=&r")
4812         (ASHIFT:GPI
4813           (match_operand:GPI 1 "register_operand" "r")
4814           (minus:QI (match_operand 2 "const_int_operand" "n")
4815                     (match_operand:QI 3 "register_operand" "r"))))]
4816   "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4817   "#"
4818   "&& true"
4819   [(const_int 0)]
4820   {
4821     rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
4822
4823     rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4824                : gen_lowpart (SImode, operands[0]));
4825
4826     emit_insn (gen_negsi2 (tmp, subreg_tmp));
4827
4828     rtx and_op = gen_rtx_AND (SImode, tmp,
4829                               GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
4830
4831     rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
4832
4833     emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
4834     DONE;
4835   }
4836   [(set_attr "length" "8")]
4837 )
4838
4839 ;; Logical left shift using SISD or Integer instruction
4840 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
4841   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
4842         (ashift:GPI
4843           (match_operand:GPI 1 "register_operand" "r,r,w,w")
4844           (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
4845   ""
4846   "@
4847    lsl\t%<w>0, %<w>1, %2
4848    lsl\t%<w>0, %<w>1, %<w>2
4849    shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4850    ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
4851   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
4852    (set_attr "arch" "*,*,simd,simd")]
4853 )
4854
4855 ;; Logical right shift using SISD or Integer instruction
4856 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
4857   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4858         (lshiftrt:GPI
4859          (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4860          (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
4861                               "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4862   ""
4863   "@
4864    lsr\t%<w>0, %<w>1, %2
4865    lsr\t%<w>0, %<w>1, %<w>2
4866    ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4867    #
4868    #"
4869   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
4870    (set_attr "arch" "*,*,simd,simd,simd")]
4871 )
4872
4873 (define_split
4874   [(set (match_operand:DI 0 "aarch64_simd_register")
4875         (lshiftrt:DI
4876            (match_operand:DI 1 "aarch64_simd_register")
4877            (match_operand:QI 2 "aarch64_simd_register")))]
4878   "TARGET_SIMD && reload_completed"
4879   [(set (match_dup 3)
4880         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4881    (set (match_dup 0)
4882         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4883   {
4884     operands[3] = gen_lowpart (QImode, operands[0]);
4885   }
4886 )
4887
4888 (define_split
4889   [(set (match_operand:SI 0 "aarch64_simd_register")
4890         (lshiftrt:SI
4891            (match_operand:SI 1 "aarch64_simd_register")
4892            (match_operand:QI 2 "aarch64_simd_register")))]
4893   "TARGET_SIMD && reload_completed"
4894   [(set (match_dup 3)
4895         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4896    (set (match_dup 0)
4897         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4898   {
4899     operands[3] = gen_lowpart (QImode, operands[0]);
4900   }
4901 )
4902
4903 ;; Arithmetic right shift using SISD or Integer instruction
4904 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4905   [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4906         (ashiftrt:GPI
4907           (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4908           (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
4909                                "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4910   ""
4911   "@
4912    asr\t%<w>0, %<w>1, %2
4913    asr\t%<w>0, %<w>1, %<w>2
4914    sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4915    #
4916    #"
4917   [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
4918    (set_attr "arch" "*,*,simd,simd,simd")]
4919 )
4920
4921 (define_split
4922   [(set (match_operand:DI 0 "aarch64_simd_register")
4923         (ashiftrt:DI
4924            (match_operand:DI 1 "aarch64_simd_register")
4925            (match_operand:QI 2 "aarch64_simd_register")))]
4926   "TARGET_SIMD && reload_completed"
4927   [(set (match_dup 3)
4928         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4929    (set (match_dup 0)
4930         (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4931 {
4932   operands[3] = gen_lowpart (QImode, operands[0]);
4933 }
4934 )
4935
4936 (define_split
4937   [(set (match_operand:SI 0 "aarch64_simd_register")
4938         (ashiftrt:SI
4939            (match_operand:SI 1 "aarch64_simd_register")
4940            (match_operand:QI 2 "aarch64_simd_register")))]
4941   "TARGET_SIMD && reload_completed"
4942   [(set (match_dup 3)
4943         (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4944    (set (match_dup 0)
4945         (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4946 {
4947   operands[3] = gen_lowpart (QImode, operands[0]);
4948 }
4949 )
4950
4951 (define_insn "*aarch64_sisd_ushl"
4952   [(set (match_operand:DI 0 "register_operand" "=w")
4953         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4954                     (match_operand:QI 2 "register_operand" "w")]
4955                    UNSPEC_SISD_USHL))]
4956   "TARGET_SIMD"
4957   "ushl\t%d0, %d1, %d2"
4958   [(set_attr "type" "neon_shift_reg")]
4959 )
4960
4961 (define_insn "*aarch64_ushl_2s"
4962   [(set (match_operand:SI 0 "register_operand" "=w")
4963         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4964                     (match_operand:QI 2 "register_operand" "w")]
4965                    UNSPEC_USHL_2S))]
4966   "TARGET_SIMD"
4967   "ushl\t%0.2s, %1.2s, %2.2s"
4968   [(set_attr "type" "neon_shift_reg")]
4969 )
4970
4971 (define_insn "*aarch64_sisd_sshl"
4972   [(set (match_operand:DI 0 "register_operand" "=w")
4973         (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4974                     (match_operand:QI 2 "register_operand" "w")]
4975                    UNSPEC_SISD_SSHL))]
4976   "TARGET_SIMD"
4977   "sshl\t%d0, %d1, %d2"
4978   [(set_attr "type" "neon_shift_reg")]
4979 )
4980
4981 (define_insn "*aarch64_sshl_2s"
4982   [(set (match_operand:SI 0 "register_operand" "=w")
4983         (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4984                     (match_operand:QI 2 "register_operand" "w")]
4985                    UNSPEC_SSHL_2S))]
4986   "TARGET_SIMD"
4987   "sshl\t%0.2s, %1.2s, %2.2s"
4988   [(set_attr "type" "neon_shift_reg")]
4989 )
4990
4991 (define_insn "*aarch64_sisd_neg_qi"
4992   [(set (match_operand:QI 0 "register_operand" "=w")
4993         (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4994                    UNSPEC_SISD_NEG))]
4995   "TARGET_SIMD"
4996   "neg\t%d0, %d1"
4997   [(set_attr "type" "neon_neg")]
4998 )
4999
5000 ;; Rotate right
5001 (define_insn "*ror<mode>3_insn"
5002   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5003      (rotatert:GPI
5004        (match_operand:GPI 1 "register_operand" "r,r")
5005        (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
5006   ""
5007   "@
5008    ror\\t%<w>0, %<w>1, %2
5009    ror\\t%<w>0, %<w>1, %<w>2"
5010   [(set_attr "type" "rotate_imm,shift_reg")]
5011 )
5012
5013 ;; zero_extend version of above
5014 (define_insn "*<optab>si3_insn_uxtw"
5015   [(set (match_operand:DI 0 "register_operand" "=r,r")
5016         (zero_extend:DI (SHIFT:SI
5017          (match_operand:SI 1 "register_operand" "r,r")
5018          (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5019   ""
5020   "@
5021    <shift>\\t%w0, %w1, %2
5022    <shift>\\t%w0, %w1, %w2"
5023   [(set_attr "type" "bfx,shift_reg")]
5024 )
5025
5026 (define_insn "*<optab><mode>3_insn"
5027   [(set (match_operand:SHORT 0 "register_operand" "=r")
5028         (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5029                       (match_operand 2 "const_int_operand" "n")))]
5030   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5031 {
5032   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5033   return "<bfshift>\t%w0, %w1, %2, %3";
5034 }
5035   [(set_attr "type" "bfx")]
5036 )
5037
5038 (define_insn "*extr<mode>5_insn"
5039   [(set (match_operand:GPI 0 "register_operand" "=r")
5040         (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5041                              (match_operand 3 "const_int_operand" "n"))
5042                  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5043                                (match_operand 4 "const_int_operand" "n"))))]
5044   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5045    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5046   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5047   [(set_attr "type" "rotate_imm")]
5048 )
5049
5050 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5051 ;; so we have to match both orderings.
5052 (define_insn "*extr<mode>5_insn_alt"
5053   [(set (match_operand:GPI 0 "register_operand" "=r")
5054         (ior:GPI  (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5055                                 (match_operand 4 "const_int_operand" "n"))
5056                   (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5057                               (match_operand 3 "const_int_operand" "n"))))]
5058   "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5059    && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5060        == GET_MODE_BITSIZE (<MODE>mode))"
5061   "extr\\t%<w>0, %<w>1, %<w>2, %4"
5062   [(set_attr "type" "rotate_imm")]
5063 )
5064
5065 ;; zero_extend version of the above
5066 (define_insn "*extrsi5_insn_uxtw"
5067   [(set (match_operand:DI 0 "register_operand" "=r")
5068         (zero_extend:DI
5069          (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5070                             (match_operand 3 "const_int_operand" "n"))
5071                  (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5072                               (match_operand 4 "const_int_operand" "n")))))]
5073   "UINTVAL (operands[3]) < 32 &&
5074    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5075   "extr\\t%w0, %w1, %w2, %4"
5076   [(set_attr "type" "rotate_imm")]
5077 )
5078
5079 (define_insn "*extrsi5_insn_uxtw_alt"
5080   [(set (match_operand:DI 0 "register_operand" "=r")
5081         (zero_extend:DI
5082          (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5083                                (match_operand 4 "const_int_operand" "n"))
5084                  (ashift:SI (match_operand:SI 1 "register_operand" "r")
5085                             (match_operand 3 "const_int_operand" "n")))))]
5086   "UINTVAL (operands[3]) < 32 &&
5087    (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5088   "extr\\t%w0, %w1, %w2, %4"
5089   [(set_attr "type" "rotate_imm")]
5090 )
5091
5092 (define_insn "*ror<mode>3_insn"
5093   [(set (match_operand:GPI 0 "register_operand" "=r")
5094         (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5095                     (match_operand 2 "const_int_operand" "n")))]
5096   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5097 {
5098   operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5099   return "ror\\t%<w>0, %<w>1, %3";
5100 }
5101   [(set_attr "type" "rotate_imm")]
5102 )
5103
5104 ;; zero_extend version of the above
5105 (define_insn "*rorsi3_insn_uxtw"
5106   [(set (match_operand:DI 0 "register_operand" "=r")
5107         (zero_extend:DI
5108          (rotate:SI (match_operand:SI 1 "register_operand" "r")
5109                     (match_operand 2 "const_int_operand" "n"))))]
5110   "UINTVAL (operands[2]) < 32"
5111 {
5112   operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5113   return "ror\\t%w0, %w1, %3";
5114 }
5115   [(set_attr "type" "rotate_imm")]
5116 )
5117
5118 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5119   [(set (match_operand:GPI 0 "register_operand" "=r")
5120         (ANY_EXTEND:GPI
5121          (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5122                        (match_operand 2 "const_int_operand" "n"))))]
5123   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5124 {
5125   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5126   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5127 }
5128   [(set_attr "type" "bfx")]
5129 )
5130
5131 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5132   [(set (match_operand:GPI 0 "register_operand" "=r")
5133         (zero_extend:GPI
5134          (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5135                          (match_operand 2 "const_int_operand" "n"))))]
5136   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5137 {
5138   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5139   return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5140 }
5141   [(set_attr "type" "bfx")]
5142 )
5143
5144 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5145   [(set (match_operand:GPI 0 "register_operand" "=r")
5146         (sign_extend:GPI
5147          (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5148                          (match_operand 2 "const_int_operand" "n"))))]
5149   "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5150 {
5151   operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5152   return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5153 }
5154   [(set_attr "type" "bfx")]
5155 )
5156
5157 ;; -------------------------------------------------------------------
5158 ;; Bitfields
5159 ;; -------------------------------------------------------------------
5160
5161 (define_expand "<optab>"
5162   [(set (match_operand:DI 0 "register_operand" "=r")
5163         (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5164                         (match_operand 2
5165                           "aarch64_simd_shift_imm_offset_di")
5166                         (match_operand 3 "aarch64_simd_shift_imm_di")))]
5167   ""
5168   {
5169     if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5170                    1, GET_MODE_BITSIZE (DImode) - 1))
5171      FAIL;
5172   }
5173 )
5174
5175
5176 (define_insn "*<optab><mode>"
5177   [(set (match_operand:GPI 0 "register_operand" "=r")
5178         (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5179                          (match_operand 2
5180                            "aarch64_simd_shift_imm_offset_<mode>" "n")
5181                          (match_operand 3
5182                            "aarch64_simd_shift_imm_<mode>" "n")))]
5183   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5184              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5185   "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5186   [(set_attr "type" "bfx")]
5187 )
5188
5189 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5190 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5191 (define_split
5192   [(set (match_operand:DI 0 "register_operand")
5193         (zero_extract:DI (match_operand:DI 1 "register_operand")
5194                          (match_operand 2
5195                            "aarch64_simd_shift_imm_offset_di")
5196                          (match_operand 3
5197                            "aarch64_simd_shift_imm_di")))]
5198   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5199              GET_MODE_BITSIZE (DImode) - 1)
5200    && (INTVAL (operands[2]) + INTVAL (operands[3]))
5201        == GET_MODE_BITSIZE (SImode)"
5202   [(set (match_dup 0)
5203         (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5204   {
5205     operands[4] = gen_lowpart (SImode, operands[1]);
5206   }
5207 )
5208
5209 ;; Bitfield Insert (insv)
5210 (define_expand "insv<mode>"
5211   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5212                           (match_operand 1 "const_int_operand")
5213                           (match_operand 2 "const_int_operand"))
5214         (match_operand:GPI 3 "general_operand"))]
5215   ""
5216 {
5217   unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5218   unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5219   rtx value = operands[3];
5220
5221   if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5222     FAIL;
5223
5224   if (CONST_INT_P (value))
5225     {
5226       unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5227
5228       /* Prefer AND/OR for inserting all zeros or all ones.  */
5229       if ((UINTVAL (value) & mask) == 0
5230            || (UINTVAL (value) & mask) == mask)
5231         FAIL;
5232
5233       /* 16-bit aligned 16-bit wide insert is handled by insv_imm.  */
5234       if (width == 16 && (pos % 16) == 0)
5235         DONE;
5236     }
5237   operands[3] = force_reg (<MODE>mode, value);
5238 })
5239
5240 (define_insn "*insv_reg<mode>"
5241   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5242                           (match_operand 1 "const_int_operand" "n")
5243                           (match_operand 2 "const_int_operand" "n"))
5244         (match_operand:GPI 3 "register_operand" "r"))]
5245   "!(UINTVAL (operands[1]) == 0
5246      || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5247          > GET_MODE_BITSIZE (<MODE>mode)))"
5248   "bfi\\t%<w>0, %<w>3, %2, %1"
5249   [(set_attr "type" "bfm")]
5250 )
5251
5252 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5253   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5254                           (match_operand 1 "const_int_operand" "n")
5255                           (match_operand 2 "const_int_operand" "n"))
5256         (zero_extend:GPI (match_operand:ALLX 3  "register_operand" "r")))]
5257   "UINTVAL (operands[1]) <= <ALLX:sizen>"
5258   "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5259   [(set_attr "type" "bfm")]
5260 )
5261
5262 (define_insn "*extr_insv_lower_reg<mode>"
5263   [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5264                           (match_operand 1 "const_int_operand" "n")
5265                           (const_int 0))
5266         (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5267                           (match_dup 1)
5268                           (match_operand 3 "const_int_operand" "n")))]
5269   "!(UINTVAL (operands[1]) == 0
5270      || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5271          > GET_MODE_BITSIZE (<MODE>mode)))"
5272   "bfxil\\t%<w>0, %<w>2, %3, %1"
5273   [(set_attr "type" "bfm")]
5274 )
5275
5276 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5277   [(set (match_operand:GPI 0 "register_operand" "=r")
5278         (ashift:GPI (ANY_EXTEND:GPI
5279                      (match_operand:ALLX 1 "register_operand" "r"))
5280                     (match_operand 2 "const_int_operand" "n")))]
5281   "UINTVAL (operands[2]) < <GPI:sizen>"
5282 {
5283   operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5284               ? GEN_INT (<ALLX:sizen>)
5285               : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5286   return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5287 }
5288   [(set_attr "type" "bfx")]
5289 )
5290
5291 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5292
5293 (define_insn "*andim_ashift<mode>_bfiz"
5294   [(set (match_operand:GPI 0 "register_operand" "=r")
5295         (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5296                              (match_operand 2 "const_int_operand" "n"))
5297                  (match_operand 3 "const_int_operand" "n")))]
5298   "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5299   "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5300   [(set_attr "type" "bfx")]
5301 )
5302
5303 ;; Match sbfiz pattern in a shift left + shift right operation.
5304
5305 (define_insn "*ashift<mode>_extv_bfiz"
5306   [(set (match_operand:GPI 0 "register_operand" "=r")
5307         (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5308                                       (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5309                                       (const_int 0))
5310                      (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5311   "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5312              1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5313   "sbfiz\\t%<w>0, %<w>1, %3, %2"
5314   [(set_attr "type" "bfx")]
5315 )
5316
5317 ;; When the bit position and width of the equivalent extraction add up to 32
5318 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5319 ;; zero-extension of the X-reg.
5320 (define_split
5321   [(set (match_operand:DI 0 "register_operand")
5322         (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5323                              (match_operand 2 "const_int_operand"))
5324                  (match_operand 3 "const_int_operand")))]
5325  "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5326   && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5327       == GET_MODE_BITSIZE (SImode)"
5328   [(set (match_dup 0)
5329         (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5330   {
5331     operands[4] = gen_lowpart (SImode, operands[1]);
5332   }
5333 )
5334
5335 (define_insn "bswap<mode>2"
5336   [(set (match_operand:GPI 0 "register_operand" "=r")
5337         (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5338   ""
5339   "rev\\t%<w>0, %<w>1"
5340   [(set_attr "type" "rev")]
5341 )
5342
5343 (define_insn "bswaphi2"
5344   [(set (match_operand:HI 0 "register_operand" "=r")
5345         (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5346   ""
5347   "rev16\\t%w0, %w1"
5348   [(set_attr "type" "rev")]
5349 )
5350
5351 (define_insn "*aarch64_bfxil<mode>"
5352   [(set (match_operand:GPI 0 "register_operand" "=r,r")
5353     (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0")
5354                     (match_operand:GPI 3 "const_int_operand" "n, Ulc"))
5355             (and:GPI (match_operand:GPI 2 "register_operand" "0,r")
5356                     (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))]
5357   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5358   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5359     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5360   {
5361     switch (which_alternative)
5362     {
5363       case 0:
5364         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5365         return "bfxil\\t%<w>0, %<w>1, 0, %3";
5366       case 1:
5367         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5368         return "bfxil\\t%<w>0, %<w>2, 0, %3";
5369       default:
5370         gcc_unreachable ();
5371     }
5372   }
5373   [(set_attr "type" "bfm")]
5374 )
5375
5376 ; Zero-extended version of above (aarch64_bfxil)
5377 (define_insn "*aarch64_bfxilsi_uxtw"
5378   [(set (match_operand:DI 0 "register_operand" "=r,r")
5379         (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand"
5380                                         "r,0")
5381                     (match_operand:SI 3 "const_int_operand" "n, Ulc"))
5382             (and:SI (match_operand:SI 2 "register_operand" "0,r")
5383                     (match_operand:SI 4 "const_int_operand" "Ulc, n")))))]
5384   "(INTVAL (operands[3]) == ~INTVAL (operands[4]))
5385   && (aarch64_high_bits_all_ones_p (INTVAL (operands[3]))
5386     || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))"
5387   {
5388     switch (which_alternative)
5389     {
5390       case 0:
5391         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3])));
5392         return "bfxil\\t%0, %1, 0, %3";
5393       case 1:
5394         operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4])));
5395         return "bfxil\\t%0, %2, 0, %3";
5396       default:
5397         gcc_unreachable ();
5398     }
5399   }
5400   [(set_attr "type" "bfm")]
5401 )
5402
5403 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5404 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5405 ;; each valid permutation.
5406
5407 (define_insn "rev16<mode>2"
5408   [(set (match_operand:GPI 0 "register_operand" "=r")
5409         (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5410                                       (const_int 8))
5411                           (match_operand:GPI 3 "const_int_operand" "n"))
5412                  (and:GPI (lshiftrt:GPI (match_dup 1)
5413                                         (const_int 8))
5414                           (match_operand:GPI 2 "const_int_operand" "n"))))]
5415   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5416    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5417   "rev16\\t%<w>0, %<w>1"
5418   [(set_attr "type" "rev")]
5419 )
5420
5421 (define_insn "rev16<mode>2_alt"
5422   [(set (match_operand:GPI 0 "register_operand" "=r")
5423         (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5424                                         (const_int 8))
5425                           (match_operand:GPI 2 "const_int_operand" "n"))
5426                  (and:GPI (ashift:GPI (match_dup 1)
5427                                       (const_int 8))
5428                           (match_operand:GPI 3 "const_int_operand" "n"))))]
5429   "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5430    && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5431   "rev16\\t%<w>0, %<w>1"
5432   [(set_attr "type" "rev")]
5433 )
5434
5435 ;; zero_extend version of above
5436 (define_insn "*bswapsi2_uxtw"
5437   [(set (match_operand:DI 0 "register_operand" "=r")
5438         (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5439   ""
5440   "rev\\t%w0, %w1"
5441   [(set_attr "type" "rev")]
5442 )
5443
5444 ;; -------------------------------------------------------------------
5445 ;; Floating-point intrinsics
5446 ;; -------------------------------------------------------------------
5447
5448 ;; frint floating-point round to integral standard patterns.
5449 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5450
5451 (define_insn "<frint_pattern><mode>2"
5452   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5453         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5454          FRINT))]
5455   "TARGET_FLOAT"
5456   "frint<frint_suffix>\\t%<s>0, %<s>1"
5457   [(set_attr "type" "f_rint<stype>")]
5458 )
5459
5460 ;; frcvt floating-point round to integer and convert standard patterns.
5461 ;; Expands to lbtrunc, lceil, lfloor, lround.
5462 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5463   [(set (match_operand:GPI 0 "register_operand" "=r")
5464         (FIXUORS:GPI
5465           (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5466            FCVT)))]
5467   "TARGET_FLOAT"
5468   "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5469   [(set_attr "type" "f_cvtf2i")]
5470 )
5471
5472 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5473   [(set (match_operand:GPI 0 "register_operand" "=r")
5474         (FIXUORS:GPI
5475           (mult:GPF
5476             (match_operand:GPF 1 "register_operand" "w")
5477             (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5478   "TARGET_FLOAT
5479    && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5480                 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5481   {
5482     int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5483     char buf[64];
5484     snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5485     output_asm_insn (buf, operands);
5486     return "";
5487   }
5488   [(set_attr "type" "f_cvtf2i")]
5489 )
5490
5491 ;; fma - expand fma into patterns with the accumulator operand first since
5492 ;; reusing the accumulator results in better register allocation.
5493 ;; The register allocator considers copy preferences in operand order,
5494 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5495
5496 (define_expand "fma<mode>4"
5497   [(set (match_operand:GPF_F16 0 "register_operand")
5498         (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5499                      (match_operand:GPF_F16 2 "register_operand")
5500                      (match_operand:GPF_F16 3 "register_operand")))]
5501   "TARGET_FLOAT"
5502 )
5503
5504 (define_insn "*aarch64_fma<mode>4"
5505   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5506         (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5507                      (match_operand:GPF_F16 3 "register_operand" "w")
5508                      (match_operand:GPF_F16 1 "register_operand" "w")))]
5509   "TARGET_FLOAT"
5510   "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5511   [(set_attr "type" "fmac<stype>")]
5512 )
5513
5514 (define_expand "fnma<mode>4"
5515   [(set (match_operand:GPF_F16 0 "register_operand")
5516         (fma:GPF_F16
5517           (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5518           (match_operand:GPF_F16 2 "register_operand")
5519           (match_operand:GPF_F16 3 "register_operand")))]
5520   "TARGET_FLOAT"
5521 )
5522
5523 (define_insn "*aarch64_fnma<mode>4"
5524   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5525         (fma:GPF_F16
5526           (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5527           (match_operand:GPF_F16 3 "register_operand" "w")
5528           (match_operand:GPF_F16 1 "register_operand" "w")))]
5529   "TARGET_FLOAT"
5530   "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5531   [(set_attr "type" "fmac<stype>")]
5532 )
5533
5534
5535 (define_expand "fms<mode>4"
5536   [(set (match_operand:GPF 0 "register_operand")
5537         (fma:GPF (match_operand:GPF 1 "register_operand")
5538                  (match_operand:GPF 2 "register_operand")
5539                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5540   "TARGET_FLOAT"
5541 )
5542
5543 (define_insn "*aarch64_fms<mode>4"
5544   [(set (match_operand:GPF 0 "register_operand" "=w")
5545         (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5546                  (match_operand:GPF 3 "register_operand" "w")
5547                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5548   "TARGET_FLOAT"
5549   "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5550   [(set_attr "type" "fmac<s>")]
5551 )
5552
5553 (define_expand "fnms<mode>4"
5554   [(set (match_operand:GPF 0 "register_operand")
5555         (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5556                  (match_operand:GPF 2 "register_operand")
5557                  (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5558   "TARGET_FLOAT"
5559 )
5560
5561 (define_insn "*aarch64_fnms<mode>4"
5562   [(set (match_operand:GPF 0 "register_operand" "=w")
5563         (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5564                  (match_operand:GPF 3 "register_operand" "w")
5565                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5566   "TARGET_FLOAT"
5567   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5568   [(set_attr "type" "fmac<s>")]
5569 )
5570
5571 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5572 (define_insn "*aarch64_fnmadd<mode>4"
5573   [(set (match_operand:GPF 0 "register_operand" "=w")
5574         (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5575                           (match_operand:GPF 3 "register_operand" "w")
5576                           (match_operand:GPF 1 "register_operand" "w"))))]
5577   "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5578   "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5579   [(set_attr "type" "fmac<s>")]
5580 )
5581
5582 ;; -------------------------------------------------------------------
5583 ;; Floating-point conversions
5584 ;; -------------------------------------------------------------------
5585
5586 (define_insn "extendsfdf2"
5587   [(set (match_operand:DF 0 "register_operand" "=w")
5588         (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5589   "TARGET_FLOAT"
5590   "fcvt\\t%d0, %s1"
5591   [(set_attr "type" "f_cvt")]
5592 )
5593
5594 (define_insn "extendhfsf2"
5595   [(set (match_operand:SF 0 "register_operand" "=w")
5596         (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5597   "TARGET_FLOAT"
5598   "fcvt\\t%s0, %h1"
5599   [(set_attr "type" "f_cvt")]
5600 )
5601
5602 (define_insn "extendhfdf2"
5603   [(set (match_operand:DF 0 "register_operand" "=w")
5604         (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5605   "TARGET_FLOAT"
5606   "fcvt\\t%d0, %h1"
5607   [(set_attr "type" "f_cvt")]
5608 )
5609
5610 (define_insn "truncdfsf2"
5611   [(set (match_operand:SF 0 "register_operand" "=w")
5612         (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5613   "TARGET_FLOAT"
5614   "fcvt\\t%s0, %d1"
5615   [(set_attr "type" "f_cvt")]
5616 )
5617
5618 (define_insn "truncsfhf2"
5619   [(set (match_operand:HF 0 "register_operand" "=w")
5620         (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5621   "TARGET_FLOAT"
5622   "fcvt\\t%h0, %s1"
5623   [(set_attr "type" "f_cvt")]
5624 )
5625
5626 (define_insn "truncdfhf2"
5627   [(set (match_operand:HF 0 "register_operand" "=w")
5628         (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5629   "TARGET_FLOAT"
5630   "fcvt\\t%h0, %d1"
5631   [(set_attr "type" "f_cvt")]
5632 )
5633
5634 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5635 ;; and making r = w more expensive
5636
5637 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5638   [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5639         (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5640   "TARGET_FLOAT"
5641   "@
5642    fcvtz<su>\t%<s>0, %<s>1
5643    fcvtz<su>\t%<w>0, %<s>1"
5644   [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5645 )
5646
5647 ;; Convert HF -> SI or DI
5648
5649 (define_insn "<optab>_trunchf<GPI:mode>2"
5650   [(set (match_operand:GPI 0 "register_operand" "=r")
5651         (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5652   "TARGET_FP_F16INST"
5653   "fcvtz<su>\t%<w>0, %h1"
5654   [(set_attr "type" "f_cvtf2i")]
5655 )
5656
5657 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5658 ;; input in a fp register and output in a integer register
5659
5660 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5661   [(set (match_operand:GPI 0 "register_operand" "=r")
5662         (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5663   "TARGET_FLOAT"
5664   "fcvtz<su>\t%<w>0, %<fpw>1"
5665   [(set_attr "type" "f_cvtf2i")]
5666 )
5667
5668 (define_insn "*fix_to_zero_extend<mode>di2"
5669   [(set (match_operand:DI 0 "register_operand" "=r")
5670         (zero_extend:DI
5671          (unsigned_fix:SI
5672           (match_operand:GPF 1 "register_operand" "w"))))]
5673   "TARGET_FLOAT"
5674   "fcvtzu\t%w0, %<s>1"
5675   [(set_attr "type" "f_cvtf2i")]
5676 )
5677
5678 (define_insn "<optab><fcvt_target><GPF:mode>2"
5679   [(set (match_operand:GPF 0 "register_operand" "=w,w")
5680         (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
5681   "TARGET_FLOAT"
5682   "@
5683    <su_optab>cvtf\t%<GPF:s>0, %<s>1
5684    <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5685   [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
5686    (set_attr "arch" "simd,fp")]
5687 )
5688
5689 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5690   [(set (match_operand:GPF 0 "register_operand" "=w")
5691         (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5692   "TARGET_FLOAT"
5693   "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5694   [(set_attr "type" "f_cvti2f")]
5695 )
5696
5697 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5698 ;; midend will arrange for an SImode conversion to HFmode to first go
5699 ;; through DFmode, then to HFmode.  But first it will try converting
5700 ;; to DImode then down, which would match our DImode pattern below and
5701 ;; give very poor code-generation.  So, we must provide our own emulation
5702 ;; of the mid-end logic.
5703
5704 (define_insn "aarch64_fp16_<optab><mode>hf2"
5705   [(set (match_operand:HF 0 "register_operand" "=w")
5706         (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5707   "TARGET_FP_F16INST"
5708   "<su_optab>cvtf\t%h0, %<w>1"
5709   [(set_attr "type" "f_cvti2f")]
5710 )
5711
5712 (define_expand "<optab>sihf2"
5713   [(set (match_operand:HF 0 "register_operand")
5714         (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5715   "TARGET_FLOAT"
5716 {
5717   if (TARGET_FP_F16INST)
5718     emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5719   else
5720     {
5721       rtx convert_target = gen_reg_rtx (DFmode);
5722       emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5723       emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5724     }
5725   DONE;
5726 }
5727 )
5728
5729 ;; For DImode there is no wide enough floating-point mode that we
5730 ;; can convert through natively (TFmode would work, but requires a library
5731 ;; call).  However, we know that any value >= 65504 will be rounded
5732 ;; to infinity on conversion.  This is well within the range of SImode, so
5733 ;; we can:
5734 ;;   Saturate to SImode.
5735 ;;   Convert from that to DFmode
5736 ;;   Convert from that to HFmode (phew!).
5737 ;; Note that the saturation to SImode requires the SIMD extensions.  If
5738 ;; we ever need to provide this pattern where the SIMD extensions are not
5739 ;; available, we would need a different approach.
5740
5741 (define_expand "<optab>dihf2"
5742   [(set (match_operand:HF 0 "register_operand")
5743         (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5744   "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5745 {
5746   if (TARGET_FP_F16INST)
5747     emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5748   else
5749     {
5750       rtx sat_target = gen_reg_rtx (SImode);
5751       emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5752       emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5753     }
5754
5755   DONE;
5756 }
5757 )
5758
5759 ;; Convert between fixed-point and floating-point (scalar modes)
5760
5761 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
5762   [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
5763         (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
5764                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5765          FCVT_F2FIXED))]
5766   ""
5767   "@
5768    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
5769    <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
5770   [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
5771    (set_attr "arch" "fp,simd")]
5772 )
5773
5774 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
5775   [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
5776         (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
5777                                    (match_operand:SI 2 "immediate_operand" "i, i")]
5778          FCVT_FIXED2F))]
5779   ""
5780   "@
5781    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
5782    <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
5783   [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
5784    (set_attr "arch" "fp,simd")]
5785 )
5786
5787 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
5788   [(set (match_operand:GPI 0 "register_operand" "=r")
5789         (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
5790                      (match_operand:SI 2 "immediate_operand" "i")]
5791          FCVT_F2FIXED))]
5792   "TARGET_FP_F16INST"
5793    "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
5794   [(set_attr "type" "f_cvtf2i")]
5795 )
5796
5797 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
5798   [(set (match_operand:HF 0 "register_operand" "=w")
5799         (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
5800                     (match_operand:SI 2 "immediate_operand" "i")]
5801          FCVT_FIXED2F))]
5802   "TARGET_FP_F16INST"
5803   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
5804   [(set_attr "type" "f_cvti2f")]
5805 )
5806
5807 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
5808   [(set (match_operand:HI 0 "register_operand" "=w")
5809         (unspec:HI [(match_operand:HF 1 "register_operand" "w")
5810                     (match_operand:SI 2 "immediate_operand" "i")]
5811          FCVT_F2FIXED))]
5812   "TARGET_SIMD"
5813   "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
5814   [(set_attr "type" "neon_fp_to_int_s")]
5815 )
5816
5817 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
5818   [(set (match_operand:HF 0 "register_operand" "=w")
5819         (unspec:HF [(match_operand:HI 1 "register_operand" "w")
5820                     (match_operand:SI 2 "immediate_operand" "i")]
5821          FCVT_FIXED2F))]
5822   "TARGET_SIMD"
5823   "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
5824   [(set_attr "type" "neon_int_to_fp_s")]
5825 )
5826
5827 ;; -------------------------------------------------------------------
5828 ;; Floating-point arithmetic
5829 ;; -------------------------------------------------------------------
5830
5831 (define_insn "add<mode>3"
5832   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5833         (plus:GPF_F16
5834          (match_operand:GPF_F16 1 "register_operand" "w")
5835          (match_operand:GPF_F16 2 "register_operand" "w")))]
5836   "TARGET_FLOAT"
5837   "fadd\\t%<s>0, %<s>1, %<s>2"
5838   [(set_attr "type" "fadd<stype>")]
5839 )
5840
5841 (define_insn "sub<mode>3"
5842   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5843         (minus:GPF_F16
5844          (match_operand:GPF_F16 1 "register_operand" "w")
5845          (match_operand:GPF_F16 2 "register_operand" "w")))]
5846   "TARGET_FLOAT"
5847   "fsub\\t%<s>0, %<s>1, %<s>2"
5848   [(set_attr "type" "fadd<stype>")]
5849 )
5850
5851 (define_insn "mul<mode>3"
5852   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5853         (mult:GPF_F16
5854          (match_operand:GPF_F16 1 "register_operand" "w")
5855          (match_operand:GPF_F16 2 "register_operand" "w")))]
5856   "TARGET_FLOAT"
5857   "fmul\\t%<s>0, %<s>1, %<s>2"
5858   [(set_attr "type" "fmul<stype>")]
5859 )
5860
5861 (define_insn "*fnmul<mode>3"
5862   [(set (match_operand:GPF 0 "register_operand" "=w")
5863         (mult:GPF
5864                  (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
5865                  (match_operand:GPF 2 "register_operand" "w")))]
5866   "TARGET_FLOAT && !flag_rounding_math"
5867   "fnmul\\t%<s>0, %<s>1, %<s>2"
5868   [(set_attr "type" "fmul<s>")]
5869 )
5870
5871 (define_insn "*fnmul<mode>3"
5872   [(set (match_operand:GPF 0 "register_operand" "=w")
5873         (neg:GPF (mult:GPF
5874                  (match_operand:GPF 1 "register_operand" "w")
5875                  (match_operand:GPF 2 "register_operand" "w"))))]
5876   "TARGET_FLOAT"
5877   "fnmul\\t%<s>0, %<s>1, %<s>2"
5878   [(set_attr "type" "fmul<s>")]
5879 )
5880
5881 (define_expand "div<mode>3"
5882  [(set (match_operand:GPF_F16 0 "register_operand")
5883        (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
5884                     (match_operand:GPF_F16 2 "register_operand")))]
5885  "TARGET_FLOAT"
5886 {
5887   if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
5888     DONE;
5889
5890   operands[1] = force_reg (<MODE>mode, operands[1]);
5891 })
5892
5893 (define_insn "*div<mode>3"
5894   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5895         (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
5896                      (match_operand:GPF_F16 2 "register_operand" "w")))]
5897   "TARGET_FLOAT"
5898   "fdiv\\t%<s>0, %<s>1, %<s>2"
5899   [(set_attr "type" "fdiv<stype>")]
5900 )
5901
5902 (define_insn "neg<mode>2"
5903   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5904         (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5905   "TARGET_FLOAT"
5906   "fneg\\t%<s>0, %<s>1"
5907   [(set_attr "type" "ffarith<stype>")]
5908 )
5909
5910 (define_expand "sqrt<mode>2"
5911   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5912         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5913   "TARGET_FLOAT"
5914 {
5915   if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
5916     DONE;
5917 })
5918
5919 (define_insn "*sqrt<mode>2"
5920   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5921         (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5922   "TARGET_FLOAT"
5923   "fsqrt\\t%<s>0, %<s>1"
5924   [(set_attr "type" "fsqrt<stype>")]
5925 )
5926
5927 (define_insn "abs<mode>2"
5928   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5929         (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5930   "TARGET_FLOAT"
5931   "fabs\\t%<s>0, %<s>1"
5932   [(set_attr "type" "ffarith<stype>")]
5933 )
5934
5935 ;; Given that smax/smin do not specify the result when either input is NaN,
5936 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
5937 ;; for smin.
5938
5939 (define_insn "smax<mode>3"
5940   [(set (match_operand:GPF 0 "register_operand" "=w")
5941         (smax:GPF (match_operand:GPF 1 "register_operand" "w")
5942                   (match_operand:GPF 2 "register_operand" "w")))]
5943   "TARGET_FLOAT"
5944   "fmaxnm\\t%<s>0, %<s>1, %<s>2"
5945   [(set_attr "type" "f_minmax<s>")]
5946 )
5947
5948 (define_insn "smin<mode>3"
5949   [(set (match_operand:GPF 0 "register_operand" "=w")
5950         (smin:GPF (match_operand:GPF 1 "register_operand" "w")
5951                   (match_operand:GPF 2 "register_operand" "w")))]
5952   "TARGET_FLOAT"
5953   "fminnm\\t%<s>0, %<s>1, %<s>2"
5954   [(set_attr "type" "f_minmax<s>")]
5955 )
5956
5957 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
5958 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
5959 ;; which implement the IEEE fmax ()/fmin () functions.
5960 (define_insn "<maxmin_uns><mode>3"
5961   [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5962         (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
5963                      (match_operand:GPF_F16 2 "register_operand" "w")]
5964                      FMAXMIN_UNS))]
5965   "TARGET_FLOAT"
5966   "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
5967   [(set_attr "type" "f_minmax<stype>")]
5968 )
5969
5970 (define_expand "lrint<GPF:mode><GPI:mode>2"
5971   [(match_operand:GPI 0 "register_operand")
5972    (match_operand:GPF 1 "register_operand")]
5973   "TARGET_FLOAT
5974    && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
5975    || !flag_trapping_math || flag_fp_int_builtin_inexact)"
5976 {
5977   rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
5978   emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
5979   emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
5980   DONE;
5981 }
5982 )
5983
5984 ;; For copysign (x, y), we want to generate:
5985 ;;
5986 ;;   LDR d2, #(1 << 63)
5987 ;;   BSL v2.8b, [y], [x]
5988 ;;
5989 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
5990 ;; aarch64_simd_bsldf will select the best suited of these instructions
5991 ;; to generate based on register allocation, and knows how to partially
5992 ;; constant fold based on the values of X and Y, so expand through that.
5993
5994 (define_expand "copysigndf3"
5995   [(match_operand:DF 0 "register_operand")
5996    (match_operand:DF 1 "register_operand")
5997    (match_operand:DF 2 "register_operand")]
5998   "TARGET_FLOAT && TARGET_SIMD"
5999 {
6000   rtx mask = gen_reg_rtx (DImode);
6001   emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
6002   emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
6003                                      operands[2], operands[1]));
6004   DONE;
6005 }
6006 )
6007
6008 ;; As above, but we must first get to a 64-bit value if we wish to use
6009 ;; aarch64_simd_bslv2sf.
6010
6011 (define_expand "copysignsf3"
6012   [(match_operand:SF 0 "register_operand")
6013    (match_operand:SF 1 "register_operand")
6014    (match_operand:SF 2 "register_operand")]
6015   "TARGET_FLOAT && TARGET_SIMD"
6016 {
6017   rtx v_bitmask = gen_reg_rtx (V2SImode);
6018
6019   /* Juggle modes to get us in to a vector mode for BSL.  */
6020   rtx op1 = lowpart_subreg (DImode, operands[1], SFmode);
6021   rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
6022   rtx tmp = gen_reg_rtx (V2SFmode);
6023   emit_move_insn (v_bitmask,
6024                   aarch64_simd_gen_const_vector_dup (V2SImode,
6025                                                      HOST_WIDE_INT_M1U << 31));
6026   emit_insn (gen_aarch64_simd_bslv2sf (tmp, v_bitmask, op2, op1));
6027   emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
6028   DONE;
6029 }
6030 )
6031
6032 ;; For xorsign (x, y), we want to generate:
6033 ;;
6034 ;; LDR   d2, #1<<63
6035 ;; AND   v3.8B, v1.8B, v2.8B
6036 ;; EOR   v0.8B, v0.8B, v3.8B
6037 ;;
6038
6039 (define_expand "xorsign<mode>3"
6040   [(match_operand:GPF 0 "register_operand")
6041    (match_operand:GPF 1 "register_operand")
6042    (match_operand:GPF 2 "register_operand")]
6043   "TARGET_FLOAT && TARGET_SIMD"
6044 {
6045
6046   machine_mode imode = <V_INT_EQUIV>mode;
6047   rtx mask = gen_reg_rtx (imode);
6048   rtx op1x = gen_reg_rtx (imode);
6049   rtx op2x = gen_reg_rtx (imode);
6050
6051   int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
6052   emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
6053                                                      imode)));
6054
6055   emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
6056                                     lowpart_subreg (imode, operands[2],
6057                                                     <MODE>mode)));
6058   emit_insn (gen_xor<v_int_equiv>3 (op1x,
6059                                     lowpart_subreg (imode, operands[1],
6060                                                     <MODE>mode),
6061                                     op2x));
6062   emit_move_insn (operands[0],
6063                   lowpart_subreg (<MODE>mode, op1x, imode));
6064   DONE;
6065 }
6066 )
6067
6068 ;; -------------------------------------------------------------------
6069 ;; Reload support
6070 ;; -------------------------------------------------------------------
6071 ;; Reload Scalar Floating point modes from constant pool.
6072 ;; The AArch64 port doesn't have __int128 constant move support.
6073 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6074  [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6075        (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6076   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6077  "TARGET_FLOAT"
6078  {
6079    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6080    emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6081    DONE;
6082  }
6083 )
6084
6085 ;; Reload Vector modes from constant pool.
6086 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6087  [(set (match_operand:VALL 0 "register_operand" "=w")
6088        (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6089   (clobber (match_operand:P 2 "register_operand" "=&r"))]
6090  "TARGET_FLOAT"
6091  {
6092    aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6093    emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6094    DONE;
6095  }
6096 )
6097
6098 (define_expand "@aarch64_reload_mov<mode>"
6099   [(set (match_operand:TX 0 "register_operand" "=w")
6100         (match_operand:TX 1 "register_operand" "w"))
6101    (clobber (match_operand:DI 2 "register_operand" "=&r"))
6102   ]
6103   "TARGET_FLOAT"
6104   {
6105     rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6106     rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6107     gen_aarch64_movtilow_tilow (op0, op1);
6108     gen_aarch64_movdi_tihigh (operands[2], op1);
6109     gen_aarch64_movtihigh_di (op0, operands[2]);
6110     DONE;
6111   }
6112 )
6113
6114 ;; The following secondary reload helpers patterns are invoked
6115 ;; after or during reload as we don't want these patterns to start
6116 ;; kicking in during the combiner.
6117
6118 (define_insn "@aarch64_movdi_<mode>low"
6119   [(set (match_operand:DI 0 "register_operand" "=r")
6120         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6121                          (const_int 64) (const_int 0)))]
6122   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6123   "fmov\\t%x0, %d1"
6124   [(set_attr "type" "f_mrc")
6125    (set_attr "length" "4")
6126   ])
6127
6128 (define_insn "@aarch64_movdi_<mode>high"
6129   [(set (match_operand:DI 0 "register_operand" "=r")
6130         (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6131                          (const_int 64) (const_int 64)))]
6132   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6133   "fmov\\t%x0, %1.d[1]"
6134   [(set_attr "type" "f_mrc")
6135    (set_attr "length" "4")
6136   ])
6137
6138 (define_insn "@aarch64_mov<mode>high_di"
6139   [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6140                          (const_int 64) (const_int 64))
6141         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6142   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6143   "fmov\\t%0.d[1], %x1"
6144   [(set_attr "type" "f_mcr")
6145    (set_attr "length" "4")
6146   ])
6147
6148 (define_insn "@aarch64_mov<mode>low_di"
6149   [(set (match_operand:TX 0 "register_operand" "=w")
6150         (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6151   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6152   "fmov\\t%d0, %x1"
6153   [(set_attr "type" "f_mcr")
6154    (set_attr "length" "4")
6155   ])
6156
6157 (define_insn "aarch64_movtilow_tilow"
6158   [(set (match_operand:TI 0 "register_operand" "=w")
6159         (zero_extend:TI
6160           (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6161   "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6162   "fmov\\t%d0, %d1"
6163   [(set_attr "type" "fmov")
6164    (set_attr "length" "4")
6165   ])
6166
6167 ;; There is a deliberate reason why the parameters of high and lo_sum's
6168 ;; don't have modes for ADRP and ADD instructions.  This is to allow high
6169 ;; and lo_sum's to be used with the labels defining the jump tables in
6170 ;; rodata section.
6171
6172 (define_expand "add_losym"
6173   [(set (match_operand 0 "register_operand" "=r")
6174         (lo_sum (match_operand 1 "register_operand" "r")
6175                 (match_operand 2 "aarch64_valid_symref" "S")))]
6176   ""
6177 {
6178   machine_mode mode = GET_MODE (operands[0]);
6179
6180   emit_insn ((mode == DImode
6181               ? gen_add_losym_di
6182               : gen_add_losym_si) (operands[0],
6183                                    operands[1],
6184                                    operands[2]));
6185   DONE;
6186 })
6187
6188 (define_insn "add_losym_<mode>"
6189   [(set (match_operand:P 0 "register_operand" "=r")
6190         (lo_sum:P (match_operand:P 1 "register_operand" "r")
6191                   (match_operand 2 "aarch64_valid_symref" "S")))]
6192   ""
6193   "add\\t%<w>0, %<w>1, :lo12:%c2"
6194   [(set_attr "type" "alu_imm")]
6195 )
6196
6197 (define_insn "ldr_got_small_<mode>"
6198   [(set (match_operand:PTR 0 "register_operand" "=r")
6199         (unspec:PTR [(mem:PTR (lo_sum:PTR
6200                               (match_operand:PTR 1 "register_operand" "r")
6201                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6202                     UNSPEC_GOTSMALLPIC))]
6203   ""
6204   "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6205   [(set_attr "type" "load_<ldst_sz>")]
6206 )
6207
6208 (define_insn "ldr_got_small_sidi"
6209   [(set (match_operand:DI 0 "register_operand" "=r")
6210         (zero_extend:DI
6211          (unspec:SI [(mem:SI (lo_sum:DI
6212                              (match_operand:DI 1 "register_operand" "r")
6213                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6214                     UNSPEC_GOTSMALLPIC)))]
6215   "TARGET_ILP32"
6216   "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6217   [(set_attr "type" "load_4")]
6218 )
6219
6220 (define_insn "ldr_got_small_28k_<mode>"
6221   [(set (match_operand:PTR 0 "register_operand" "=r")
6222         (unspec:PTR [(mem:PTR (lo_sum:PTR
6223                               (match_operand:PTR 1 "register_operand" "r")
6224                               (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6225                     UNSPEC_GOTSMALLPIC28K))]
6226   ""
6227   "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6228   [(set_attr "type" "load_<ldst_sz>")]
6229 )
6230
6231 (define_insn "ldr_got_small_28k_sidi"
6232   [(set (match_operand:DI 0 "register_operand" "=r")
6233         (zero_extend:DI
6234          (unspec:SI [(mem:SI (lo_sum:DI
6235                              (match_operand:DI 1 "register_operand" "r")
6236                              (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6237                     UNSPEC_GOTSMALLPIC28K)))]
6238   "TARGET_ILP32"
6239   "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6240   [(set_attr "type" "load_4")]
6241 )
6242
6243 (define_insn "ldr_got_tiny"
6244   [(set (match_operand:DI 0 "register_operand" "=r")
6245         (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6246                    UNSPEC_GOTTINYPIC))]
6247   ""
6248   "ldr\\t%0, %L1"
6249   [(set_attr "type" "load_8")]
6250 )
6251
6252 (define_insn "aarch64_load_tp_hard"
6253   [(set (match_operand:DI 0 "register_operand" "=r")
6254         (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6255   ""
6256   "mrs\\t%0, tpidr_el0"
6257   [(set_attr "type" "mrs")]
6258 )
6259
6260 ;; The TLS ABI specifically requires that the compiler does not schedule
6261 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6262 ;; Therefore we treat the stubs as an atomic sequence.
6263 (define_expand "tlsgd_small_<mode>"
6264  [(parallel [(set (match_operand 0 "register_operand" "")
6265                   (call (mem:DI (match_dup 2)) (const_int 1)))
6266              (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
6267              (clobber (reg:DI LR_REGNUM))])]
6268  ""
6269 {
6270   operands[2] = aarch64_tls_get_addr ();
6271 })
6272
6273 (define_insn "*tlsgd_small_<mode>"
6274   [(set (match_operand 0 "register_operand" "")
6275         (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6276    (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6277    (clobber (reg:DI LR_REGNUM))
6278   ]
6279   ""
6280   "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6281   [(set_attr "type" "call")
6282    (set_attr "length" "16")])
6283
6284 (define_insn "tlsie_small_<mode>"
6285   [(set (match_operand:PTR 0 "register_operand" "=r")
6286         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6287                    UNSPEC_GOTSMALLTLS))]
6288   ""
6289   "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6290   [(set_attr "type" "load_4")
6291    (set_attr "length" "8")]
6292 )
6293
6294 (define_insn "tlsie_small_sidi"
6295   [(set (match_operand:DI 0 "register_operand" "=r")
6296         (zero_extend:DI
6297           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6298                       UNSPEC_GOTSMALLTLS)))]
6299   ""
6300   "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6301   [(set_attr "type" "load_4")
6302    (set_attr "length" "8")]
6303 )
6304
6305 (define_insn "tlsie_tiny_<mode>"
6306   [(set (match_operand:PTR 0 "register_operand" "=&r")
6307         (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6308                      (match_operand:PTR 2 "register_operand" "r")]
6309                    UNSPEC_GOTTINYTLS))]
6310   ""
6311   "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6312   [(set_attr "type" "multiple")
6313    (set_attr "length" "8")]
6314 )
6315
6316 (define_insn "tlsie_tiny_sidi"
6317   [(set (match_operand:DI 0 "register_operand" "=&r")
6318         (zero_extend:DI
6319           (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6320                       (match_operand:DI 2 "register_operand" "r")
6321                       ]
6322                       UNSPEC_GOTTINYTLS)))]
6323   ""
6324   "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6325   [(set_attr "type" "multiple")
6326    (set_attr "length" "8")]
6327 )
6328
6329 (define_insn "tlsle12_<mode>"
6330   [(set (match_operand:P 0 "register_operand" "=r")
6331         (unspec:P [(match_operand:P 1 "register_operand" "r")
6332                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6333                    UNSPEC_TLSLE12))]
6334   ""
6335   "add\\t%<w>0, %<w>1, #%L2";
6336   [(set_attr "type" "alu_sreg")
6337    (set_attr "length" "4")]
6338 )
6339
6340 (define_insn "tlsle24_<mode>"
6341   [(set (match_operand:P 0 "register_operand" "=r")
6342         (unspec:P [(match_operand:P 1 "register_operand" "r")
6343                    (match_operand 2 "aarch64_tls_le_symref" "S")]
6344                    UNSPEC_TLSLE24))]
6345   ""
6346   "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6347   [(set_attr "type" "multiple")
6348    (set_attr "length" "8")]
6349 )
6350
6351 (define_insn "tlsle32_<mode>"
6352   [(set (match_operand:P 0 "register_operand" "=r")
6353         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6354                    UNSPEC_TLSLE32))]
6355   ""
6356   "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6357   [(set_attr "type" "multiple")
6358    (set_attr "length" "8")]
6359 )
6360
6361 (define_insn "tlsle48_<mode>"
6362   [(set (match_operand:P 0 "register_operand" "=r")
6363         (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6364                    UNSPEC_TLSLE48))]
6365   ""
6366   "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6367   [(set_attr "type" "multiple")
6368    (set_attr "length" "12")]
6369 )
6370
6371 (define_expand "tlsdesc_small_<mode>"
6372   [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6373   "TARGET_TLS_DESC"
6374   {
6375     if (TARGET_SVE)
6376       emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6377     else
6378       emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6379     DONE;
6380   }
6381 )
6382
6383 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6384 ;; R0 and LR.
6385 (define_insn "tlsdesc_small_advsimd_<mode>"
6386   [(set (reg:PTR R0_REGNUM)
6387         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6388                     UNSPEC_TLSDESC))
6389    (clobber (reg:DI LR_REGNUM))
6390    (clobber (reg:CC CC_REGNUM))
6391    (clobber (match_scratch:DI 1 "=r"))]
6392   "TARGET_TLS_DESC && !TARGET_SVE"
6393   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6394   [(set_attr "type" "call")
6395    (set_attr "length" "16")])
6396
6397 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6398 ;; all vector registers, and clobber all predicate registers, on
6399 ;; top of the usual R0 and LR.
6400 (define_insn "tlsdesc_small_sve_<mode>"
6401   [(set (reg:PTR R0_REGNUM)
6402         (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6403                     UNSPEC_TLSDESC))
6404    (clobber (reg:DI LR_REGNUM))
6405    (clobber (reg:CC CC_REGNUM))
6406    (clobber_high (reg:TI V0_REGNUM))
6407    (clobber_high (reg:TI V1_REGNUM))
6408    (clobber_high (reg:TI V2_REGNUM))
6409    (clobber_high (reg:TI V3_REGNUM))
6410    (clobber_high (reg:TI V4_REGNUM))
6411    (clobber_high (reg:TI V5_REGNUM))
6412    (clobber_high (reg:TI V6_REGNUM))
6413    (clobber_high (reg:TI V7_REGNUM))
6414    (clobber_high (reg:TI V8_REGNUM))
6415    (clobber_high (reg:TI V9_REGNUM))
6416    (clobber_high (reg:TI V10_REGNUM))
6417    (clobber_high (reg:TI V11_REGNUM))
6418    (clobber_high (reg:TI V12_REGNUM))
6419    (clobber_high (reg:TI V13_REGNUM))
6420    (clobber_high (reg:TI V14_REGNUM))
6421    (clobber_high (reg:TI V15_REGNUM))
6422    (clobber_high (reg:TI V16_REGNUM))
6423    (clobber_high (reg:TI V17_REGNUM))
6424    (clobber_high (reg:TI V18_REGNUM))
6425    (clobber_high (reg:TI V19_REGNUM))
6426    (clobber_high (reg:TI V20_REGNUM))
6427    (clobber_high (reg:TI V21_REGNUM))
6428    (clobber_high (reg:TI V22_REGNUM))
6429    (clobber_high (reg:TI V23_REGNUM))
6430    (clobber_high (reg:TI V24_REGNUM))
6431    (clobber_high (reg:TI V25_REGNUM))
6432    (clobber_high (reg:TI V26_REGNUM))
6433    (clobber_high (reg:TI V27_REGNUM))
6434    (clobber_high (reg:TI V28_REGNUM))
6435    (clobber_high (reg:TI V29_REGNUM))
6436    (clobber_high (reg:TI V30_REGNUM))
6437    (clobber_high (reg:TI V31_REGNUM))
6438    (clobber (reg:VNx2BI P0_REGNUM))
6439    (clobber (reg:VNx2BI P1_REGNUM))
6440    (clobber (reg:VNx2BI P2_REGNUM))
6441    (clobber (reg:VNx2BI P3_REGNUM))
6442    (clobber (reg:VNx2BI P4_REGNUM))
6443    (clobber (reg:VNx2BI P5_REGNUM))
6444    (clobber (reg:VNx2BI P6_REGNUM))
6445    (clobber (reg:VNx2BI P7_REGNUM))
6446    (clobber (reg:VNx2BI P8_REGNUM))
6447    (clobber (reg:VNx2BI P9_REGNUM))
6448    (clobber (reg:VNx2BI P10_REGNUM))
6449    (clobber (reg:VNx2BI P11_REGNUM))
6450    (clobber (reg:VNx2BI P12_REGNUM))
6451    (clobber (reg:VNx2BI P13_REGNUM))
6452    (clobber (reg:VNx2BI P14_REGNUM))
6453    (clobber (reg:VNx2BI P15_REGNUM))
6454    (clobber (match_scratch:DI 1 "=r"))]
6455   "TARGET_TLS_DESC && TARGET_SVE"
6456   "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6457   [(set_attr "type" "call")
6458    (set_attr "length" "16")])
6459
6460 (define_insn "stack_tie"
6461   [(set (mem:BLK (scratch))
6462         (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6463                      (match_operand:DI 1 "register_operand" "rk")]
6464                     UNSPEC_PRLG_STK))]
6465   ""
6466   ""
6467   [(set_attr "length" "0")]
6468 )
6469
6470 ;; Pointer authentication patterns are always provided.  In architecture
6471 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6472 ;; This lets the user write portable software which authenticates pointers
6473 ;; when run on something which implements ARMv8.3-A, and which runs
6474 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6475 ;; implemented.
6476
6477 ;; Signing/Authenticating R30 using SP as the salt.
6478
6479 (define_insn "<pauth_mnem_prefix>sp"
6480   [(set (reg:DI R30_REGNUM)
6481         (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6482   ""
6483   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
6484 )
6485
6486 ;; Signing/Authenticating X17 using X16 as the salt.
6487
6488 (define_insn "<pauth_mnem_prefix>1716"
6489   [(set (reg:DI R17_REGNUM)
6490         (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6491   ""
6492   "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
6493 )
6494
6495 ;; Stripping the signature in R30.
6496
6497 (define_insn "xpaclri"
6498   [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6499   ""
6500   "hint\t7 // xpaclri"
6501 )
6502
6503 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6504 ;; all of memory.  This blocks insns from being moved across this point.
6505
6506 (define_insn "blockage"
6507   [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6508   ""
6509   ""
6510   [(set_attr "length" "0")
6511    (set_attr "type" "block")]
6512 )
6513
6514 (define_insn "probe_stack_range"
6515   [(set (match_operand:DI 0 "register_operand" "=rk")
6516         (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6517                              (match_operand:DI 2 "register_operand" "r")]
6518                               UNSPECV_PROBE_STACK_RANGE))]
6519   ""
6520 {
6521   return aarch64_output_probe_stack_range (operands[0], operands[2]);
6522 }
6523   [(set_attr "length" "32")]
6524 )
6525
6526 ;; This instruction is used to generate the stack clash stack adjustment and
6527 ;; probing loop.  We can't change the control flow during prologue and epilogue
6528 ;; code generation.  So we must emit a volatile unspec and expand it later on.
6529
6530 (define_insn "@probe_sve_stack_clash_<mode>"
6531   [(set (match_operand:P 0 "register_operand" "=rk")
6532         (unspec_volatile:P [(match_operand:P 1 "register_operand" "0")
6533                             (match_operand:P 2 "register_operand" "r")
6534                             (match_operand:P 3 "const_int_operand" "n")
6535                             (match_operand:P 4 "aarch64_plus_immediate" "L")]
6536                              UNSPECV_PROBE_STACK_RANGE))]
6537   "TARGET_SVE"
6538 {
6539   return aarch64_output_probe_sve_stack_clash (operands[0], operands[2],
6540                                                operands[3], operands[4]);
6541 }
6542   [(set_attr "length" "28")]
6543 )
6544
6545 ;; Named pattern for expanding thread pointer reference.
6546 (define_expand "get_thread_pointerdi"
6547   [(match_operand:DI 0 "register_operand" "=r")]
6548   ""
6549 {
6550   rtx tmp = aarch64_load_tp (operands[0]);
6551   if (tmp != operands[0])
6552     emit_move_insn (operands[0], tmp);
6553   DONE;
6554 })
6555
6556 ;; Named patterns for stack smashing protection.
6557 (define_expand "stack_protect_set"
6558   [(match_operand 0 "memory_operand")
6559    (match_operand 1 "memory_operand")]
6560   ""
6561 {
6562   machine_mode mode = GET_MODE (operands[0]);
6563
6564   emit_insn ((mode == DImode
6565               ? gen_stack_protect_set_di
6566               : gen_stack_protect_set_si) (operands[0], operands[1]));
6567   DONE;
6568 })
6569
6570 (define_insn "stack_protect_set_<mode>"
6571   [(set (match_operand:PTR 0 "memory_operand" "=m")
6572         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6573          UNSPEC_SP_SET))
6574    (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6575   ""
6576   "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
6577   [(set_attr "length" "12")
6578    (set_attr "type" "multiple")])
6579
6580 (define_expand "stack_protect_test"
6581   [(match_operand 0 "memory_operand")
6582    (match_operand 1 "memory_operand")
6583    (match_operand 2)]
6584   ""
6585 {
6586   rtx result;
6587   machine_mode mode = GET_MODE (operands[0]);
6588
6589   result = gen_reg_rtx(mode);
6590
6591   emit_insn ((mode == DImode
6592               ? gen_stack_protect_test_di
6593               : gen_stack_protect_test_si) (result,
6594                                             operands[0],
6595                                             operands[1]));
6596
6597   if (mode == DImode)
6598     emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6599                                     result, const0_rtx, operands[2]));
6600   else
6601     emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6602                                     result, const0_rtx, operands[2]));
6603   DONE;
6604 })
6605
6606 (define_insn "stack_protect_test_<mode>"
6607   [(set (match_operand:PTR 0 "register_operand" "=r")
6608         (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
6609                      (match_operand:PTR 2 "memory_operand" "m")]
6610          UNSPEC_SP_TEST))
6611    (clobber (match_scratch:PTR 3 "=&r"))]
6612   ""
6613   "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
6614   [(set_attr "length" "12")
6615    (set_attr "type" "multiple")])
6616
6617 ;; Write Floating-point Control Register.
6618 (define_insn "set_fpcr"
6619   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
6620   ""
6621   "msr\\tfpcr, %0"
6622   [(set_attr "type" "mrs")])
6623
6624 ;; Read Floating-point Control Register.
6625 (define_insn "get_fpcr"
6626   [(set (match_operand:SI 0 "register_operand" "=r")
6627         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
6628   ""
6629   "mrs\\t%0, fpcr"
6630   [(set_attr "type" "mrs")])
6631
6632 ;; Write Floating-point Status Register.
6633 (define_insn "set_fpsr"
6634   [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
6635   ""
6636   "msr\\tfpsr, %0"
6637   [(set_attr "type" "mrs")])
6638
6639 ;; Read Floating-point Status Register.
6640 (define_insn "get_fpsr"
6641   [(set (match_operand:SI 0 "register_operand" "=r")
6642         (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
6643   ""
6644   "mrs\\t%0, fpsr"
6645   [(set_attr "type" "mrs")])
6646
6647
6648 ;; Define the subtract-one-and-jump insns so loop.c
6649 ;; knows what to generate.
6650 (define_expand "doloop_end"
6651   [(use (match_operand 0 "" ""))      ; loop pseudo
6652    (use (match_operand 1 "" ""))]     ; label
6653   "optimize > 0 && flag_modulo_sched"
6654 {
6655   rtx s0;
6656   rtx bcomp;
6657   rtx loc_ref;
6658   rtx cc_reg;
6659   rtx insn;
6660   rtx cmp;
6661
6662   /* Currently SMS relies on the do-loop pattern to recognize loops
6663      where (1) the control part consists of all insns defining and/or
6664      using a certain 'count' register and (2) the loop count can be
6665      adjusted by modifying this register prior to the loop.
6666      ??? The possible introduction of a new block to initialize the
6667      new IV can potentially affect branch optimizations.  */
6668
6669   if (GET_MODE (operands[0]) != DImode)
6670     FAIL;
6671
6672   s0 = operands [0];
6673   insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
6674
6675   cmp = XVECEXP (PATTERN (insn), 0, 0);
6676   cc_reg = SET_DEST (cmp);
6677   bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6678   loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6679   emit_jump_insn (gen_rtx_SET (pc_rtx,
6680                                gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6681                                                      loc_ref, pc_rtx)));
6682   DONE;
6683 })
6684
6685 ;; Track speculation through conditional branches.  We assume that
6686 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
6687 (define_insn "speculation_tracker"
6688   [(set (reg:DI SPECULATION_TRACKER_REGNUM)
6689         (unspec [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
6690          UNSPEC_SPECULATION_TRACKER))]
6691   ""
6692   {
6693     operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6694     output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
6695     return "";
6696   }
6697   [(set_attr "type" "csel")]
6698 )
6699
6700 ;; Helper for aarch64.c code.
6701 (define_expand "set_clobber_cc"
6702   [(parallel [(set (match_operand 0)
6703                    (match_operand 1))
6704               (clobber (reg:CC CC_REGNUM))])])
6705
6706 ;; Hard speculation barrier.
6707 (define_insn "speculation_barrier"
6708   [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
6709   ""
6710   "isb\;dsb\\tsy"
6711   [(set_attr "length" "8")
6712    (set_attr "type" "block")
6713    (set_attr "speculation_barrier" "true")]
6714 )
6715
6716 ;; Support for __builtin_speculation_safe_value when we have speculation
6717 ;; tracking enabled.  Use the speculation tracker to decide whether to
6718 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
6719 (define_expand "@despeculate_copy<ALLI_TI:mode>"
6720   [(set (match_operand:ALLI_TI 0 "register_operand" "=r")
6721         (unspec_volatile:ALLI_TI
6722          [(match_operand:ALLI_TI 1 "register_operand" "r")
6723           (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ")
6724           (use (reg:DI SPECULATION_TRACKER_REGNUM))
6725           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6726   ""
6727   "
6728   {
6729     if (operands[2] == const0_rtx)
6730       {
6731         rtx tracker;
6732         if (<MODE>mode == TImode)
6733           tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6734         else
6735           tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
6736
6737         emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
6738                                                  tracker));
6739         DONE;
6740       }
6741   }
6742   "
6743 )
6744
6745 ;; Patterns to match despeculate_copy<mode>.  Note that "hint 0x14" is the
6746 ;; encoding for CSDB, but will work in older versions of the assembler.
6747 (define_insn "*despeculate_copy<ALLI:mode>_insn"
6748   [(set (match_operand:ALLI 0 "register_operand" "=r")
6749         (unspec_volatile:ALLI
6750          [(match_operand:ALLI 1 "register_operand" "r")
6751           (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
6752           (use (reg:DI SPECULATION_TRACKER_REGNUM))
6753           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6754   ""
6755   {
6756     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6757     output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
6758                      operands);
6759     return "";
6760   }
6761   [(set_attr "length" "12")
6762    (set_attr "type" "block")
6763    (set_attr "speculation_barrier" "true")]
6764 )
6765
6766 ;; Pattern to match despeculate_copyti
6767 (define_insn "*despeculate_copyti_insn"
6768   [(set (match_operand:TI 0 "register_operand" "=r")
6769         (unspec_volatile:TI
6770          [(match_operand:TI 1 "register_operand" "r")
6771           (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
6772           (use (reg:DI SPECULATION_TRACKER_REGNUM))
6773           (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6774   ""
6775   {
6776     operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6777     output_asm_insn
6778       ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
6779        operands);
6780     return "";
6781   }
6782   [(set_attr "length" "16")
6783    (set_attr "type" "block")
6784    (set_attr "speculation_barrier" "true")]
6785 )
6786
6787 (define_insn "despeculate_simple<ALLI:mode>"
6788   [(set (match_operand:ALLI 0 "register_operand" "=r")
6789         (unspec_volatile:ALLI
6790          [(match_operand:ALLI 1 "register_operand" "r")
6791           (use (match_operand:ALLI 2 "register_operand" ""))]
6792          UNSPECV_SPECULATION_BARRIER))]
6793   ""
6794   "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
6795   [(set_attr "type" "block")
6796    (set_attr "length" "8")
6797    (set_attr "speculation_barrier" "true")]
6798 )
6799
6800 (define_insn "despeculate_simpleti"
6801   [(set (match_operand:TI 0 "register_operand" "=r")
6802         (unspec_volatile:TI
6803          [(match_operand:TI 1 "register_operand" "r")
6804           (use (match_operand:DI 2 "register_operand" ""))]
6805          UNSPECV_SPECULATION_BARRIER))]
6806   ""
6807   "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
6808   [(set_attr "type" "block")
6809    (set_attr "length" "12")
6810    (set_attr "speculation_barrier" "true")]
6811 )
6812
6813 ;; AdvSIMD Stuff
6814 (include "aarch64-simd.md")
6815
6816 ;; Atomic Operations
6817 (include "atomics.md")
6818
6819 ;; ldp/stp peephole patterns
6820 (include "aarch64-ldpstp.md")
6821
6822 ;; SVE.
6823 (include "aarch64-sve.md")